Merge branch 'master' into feature/ooruffus

RELEASE_NOTES.rst

@@ -3,10 +3,17 @@ RELEASE NOTES
 
 OCRmyPDF uses `semantic versioning <http://semver.org/>`_.
 
+v4.3.3:
+=======
+
+-  Fixed PDF/A creation with Ghostscript 9.20 properly
+-  Fixed an exception on inline stencil masks with a missing optional parameter
+
+
 v4.3.2:
 =======
 
--  Fixed a PDF/A creation issue with Ghostscript 9.20 
+-  Fixed a PDF/A creation issue with Ghostscript 9.20 (note: this fix did not actually work)
 
 
 v4.3.1:

ocrmypdf/__main__.py

@@ -1161,7 +1161,7 @@ def merge_pages_ghostscript(
 
     pdf_pages = sorted(input_files, key=input_file_order)
     log.debug("Final pages: " + "\n".join(pdf_pages))
-    ghostscript.generate_pdfa(pdf_pages, output_file, options.jobs or 1)
+    ghostscript.generate_pdfa(pdf_pages, output_file, log, options.jobs or 1)
 
 
 def merge_pages_qpdf(

ocrmypdf/ghostscript.py

@@ -42,7 +42,7 @@ def rasterize_pdf(input_file, output_file, xres, yres, raster_device, log,
             log.error('Ghostscript rendering failed')
 
 
-def generate_pdfa(pdf_pages, output_file, threads=1):
+def generate_pdfa(pdf_pages, output_file, log, threads=1):
     with NamedTemporaryFile(delete=True) as gs_pdf:
         args_gs = [
             get_program("gs"),
@@ -56,9 +56,32 @@ def generate_pdfa(pdf_pages, output_file, threads=1):
             "-sProcessColorModel=DeviceRGB",
             "-dJPEGQ=95",
             "-dPDFA=2",
-            "-sPDFACompatibilityPolicy=1",
+            "-dPDFACompatibilityPolicy=1",
             "-sOutputFile=" + gs_pdf.name,
         ]
         args_gs.extend(pdf_pages)
-        check_call(args_gs)
-        copy(gs_pdf.name, output_file)
+        p = Popen(args_gs, close_fds=True, stdout=PIPE, stderr=PIPE,
+                  universal_newlines=True)
+        stdout, stderr = p.communicate()
+        if stdout:
+            if 'error' in stdout:
+                log.error(stdout)
+            elif 'overprint mode not set' in stdout:
+                # Unless someone is going to print PDF/A documents on a
+                # magical sRGB printer I can't see the removal of overprinting
+                # being a problem....
+                log.debug(
+                    "Ghostscript had to remove PDF 'overprinting' from the "
+                    "input file to complete PDF/A conversion. "
+                    )
+            else:
+                log.debug(stdout)
+        if stderr:
+            log.error(stderr)
+
+        if p.returncode == 0:
+            # Ghostscript does not change return code when it fails to create
+            # PDF/A - check PDF/A status elsewhere
+            copy(gs_pdf.name, output_file)
+        else:
+            log.error('Ghostscript PDF/A failed')

tests/spoof/gs_feature_elision.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import sys
+import os
+from subprocess import check_call
+
+"""Replicate one type of Ghostscript feature elision warning during
+PDF/A creation."""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+elision_warning = """GPL Ghostscript 9.20: Setting Overprint Mode to 1
+not permitted in PDF/A-2, overprint mode not set"""
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    gs_args = ['gs'] + sys.argv[1:]
+    check_call(gs_args)
+
+    if '-sDEVICE=pdfwrite' in sys.argv[1:]:
+        print(elision_warning)
+
+    sys.exit(0)
+
+if __name__ == '__main__':
+    main()

tests/spoof/gs_pdfa_failure.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+import sys
+import os
+
+"""Replicate Ghostscript PDF/A conversion failure by suppressing some
+arguments"""
+
+
+def real_ghostscript(argv):
+    gs_args = ['gs'] + argv[1:]
+    os.execvp("gs", gs_args)
+    return  # Not reachable
+
+
+def main():
+    if '--version' in sys.argv:
+        print('9.20')
+        print('SPOOFED: ' + os.path.basename(__filename__))
+        sys.exit(0)
+
+    # Unless some argument is calling for PDFA generation, forward to
+    # real ghostscript
+    if not any(arg.startswith('-dPDFA') for arg in sys.argv):
+        real_ghostscript(sys.argv)
+        return
+
+    # Remove the two arguments that tell ghostscript to create a PDF/A
+    # Does not remove the Postscript definition file - not necessary
+    # to cause PDF/A creation failure
+    argv = []
+    for arg in sys.argv:
+        if arg.startswith('-dPDFA'):
+            continue
+        elif arg.startswith('-dPDFACompatibilityPolicy'):
+            continue
+        argv.append(arg)
+
+    real_ghostscript(argv)
+
+
+if __name__ == '__main__':
+    main()

tests/test_main.py

@@ -90,41 +90,56 @@ def run_ocrmypdf(input_basename, output_basename, *args, env=None):
     return p, out, err
 
 
-def spoof(replace_program, with_spoof):
+def spoof(**kwargs):
     """Modify environment variables to override subprocess executables
 
+    spoof(program1='replacement', ...)
+
     Before running any executable, ocrmypdf checks the environment variable
     OCRMYPDF_PROGRAMNAME to override default program name/location, e.g.
     OCRMYPDF_GS redirects from the system path Ghostscript ("gs") to elsewhere.
+
     """
     env = os.environ.copy()
-    spoofer = os.path.join(SPOOF_PATH, with_spoof)
-    if not os.access(spoofer, os.X_OK):
-        os.chmod(spoofer, 0o755)
-    env['OCRMYPDF_' + replace_program.upper()] = spoofer
+
+    for replace_program, with_spoof in kwargs.items():
+        spoofer = os.path.join(SPOOF_PATH, with_spoof)
+        if not os.access(spoofer, os.X_OK):
+            os.chmod(spoofer, 0o755)
+        env['OCRMYPDF_' + replace_program.upper()] = spoofer
     return env
 
 
 @pytest.fixture
 def spoof_tesseract_noop():
-    return spoof('tesseract', 'tesseract_noop.py')
+    return spoof(tesseract='tesseract_noop.py')
 
 
 @pytest.fixture
 def spoof_tesseract_cache():
     if running_in_docker():
         return os.environ.copy()
-    return spoof('tesseract', "tesseract_cache.py")
+    return spoof(tesseract="tesseract_cache.py")
 
 
 @pytest.fixture
 def spoof_tesseract_crash():
-    return spoof('tesseract', 'tesseract_crash.py')
+    return spoof(tesseract='tesseract_crash.py')
 
 
 @pytest.fixture
 def spoof_tesseract_big_image_error():
-    return spoof('tesseract', 'tesseract_big_image_error.py')
+    return spoof(tesseract='tesseract_big_image_error.py')
+
+
+@pytest.fixture
+def spoof_no_tess_no_pdfa():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_pdfa_failure.py')
+
+
+@pytest.fixture
+def spoof_no_tess_pdfa_warning():
+    return spoof(tesseract='tesseract_noop.py', gs='gs_feature_elision.py')
 
 
 def test_quick(spoof_tesseract_cache):
@@ -722,3 +737,14 @@ def test_rotated_skew_timeout():
         in_pageinfo['height_pixels'] == out_pageinfo['width_pixels'], \
         "Expected page rotation to be baked in"
 
+
+def test_ghostscript_pdfa_failure(spoof_no_tess_no_pdfa):
+    p, out, err = run_ocrmypdf(
+        'ccitt.pdf', 'test_pdfa_failure.pdf',
+        env=spoof_no_tess_no_pdfa)
+    assert p.returncode == 4, "Expected return code 4 when PDF/A fails"
+
+
+def test_ghostscript_feature_elision(spoof_no_tess_pdfa_warning):
+    check_ocrmypdf('ccitt.pdf', 'test_feature_elision.pdf',
+                   env=spoof_no_tess_pdfa_warning)