Merge branch 'feature/soft-error'

docs/advanced.rst

@@ -104,7 +104,7 @@ was requested, the preprocessed image layer will be inserted.
 If you want to adjust the amount of time spent on OCR, change
 ``--tesseract-timeout``. You can also automatically skip images that
 exceed a certain number of megapixels with ``--skip-big``. (A 300 DPI,
-8.5×11" page is 8.4 megapixels.)
+8.5×11" page image is 8.4 megapixels.)
 
 .. code-block:: bash
 
@@ -241,6 +241,17 @@ PDF.js viewer.
 
 This works in all versions of Tesseract.
 
+Rendering and rasterizing options
+=================================
+
+.. versionadded:: 14.3.0
+
+The ``--continue-on-soft-render-error`` option allows OCRmyPDF to
+proceed if a page cannot be rasterized rendered. This is useful if you are
+trying to get the best possible OCR from a PDF that is not well-formed,
+and you are willing to accept some pages that may not visually match the
+input, and that may not OCR well.
+
 Return code policy
 ==================
 

docs/pdfsecurity.rst

@@ -29,13 +29,9 @@ attack vectors.
 In short, PDFs `may contain
 viruses <https://security.stackexchange.com/questions/64052/can-a-pdf-file-contain-a-virus>`__.
 
-This
-`article <https://theinvisiblethings.blogspot.ca/2013/02/converting-untrusted-pdfs-into-trusted.html>`__
-describes a high-paranoia method which allows potentially hostile PDFs
-to be viewed and rasterized safely in a disposable virtual machine. A
-trusted PDF created in this manner is converted to images and loses all
-information making it searchable and losing all compression. OCRmyPDF
-could be used to restore searchability.
+If you do not trust a PDF or its source, do not open it or use OCRmyPDF
+on it. Consider using a Docker container or virtual machine to isolate
+an untrusted PDF from your system.
 
 How OCRmyPDF processes PDFs
 ===========================
@@ -43,11 +39,11 @@ How OCRmyPDF processes PDFs
 OCRmyPDF must open and interpret your PDF in order to insert an OCR
 layer. First, it runs all PDFs through
 `pikepdf <https://github.com/pikepdf/pikepdf>`__, a library based on
-`qpdf <https://github.com/qpdf/qpdf>`__, a program that repairs PDFs
+`QPDF <https://github.com/qpdf/qpdf>`__, a program that repairs PDFs
 with syntax errors. This is done because, in the author's experience, a
 significant number of PDFs in the wild, especially those created by
-scanners, are not well-formed files. qpdf makes it more likely that
-OCRmyPDF will succeed, but offers no security guarantees. qpdf is also
+scanners, are not well-formed files. QPDF makes it more likely that
+OCRmyPDF will succeed, but offers no security guarantees. QPDF is also
 used to split the PDF into single page PDFs.
 
 Finally, OCRmyPDF rasterizes each page of the PDF using
@@ -133,7 +129,7 @@ The author also provides professional services that include OCR and
 building databases around PDFs, and is happy to provide consultation.
 
 Abbyy Cloud OCR is viable commercial alternative with a web services
-API. Amazon Textract, Google Cloud Vision, and Microsoft Azure 
+API. Amazon Textract, Google Cloud Vision, and Microsoft Azure
 Computer Vision provide advanced OCR but have less PDF rendering capability.
 
 Password protection, digital signatures and certification

src/ocrmypdf/_exec/ghostscript.py

@@ -30,7 +30,7 @@ except AttributeError:
 
 log = logging.getLogger(__name__)
 
-# Most reliable what to get the bitness of Python interpreter, according to Python docs
+# Most reliable way to get the bitness of Python interpreter, according to Python docs
 _IS_64BIT = sys.maxsize > 2**32
 
 _GSWIN = None
@@ -63,6 +63,7 @@ def rasterize_pdf(
     page_dpi: Resolution | None = None,
     rotation: int | None = None,
     filter_vector: bool = False,
+    stop_on_error: bool = False,
 ):
     """Rasterize one page of a PDF at resolution raster_dpi in canvas units."""
     raster_dpi = raster_dpi.round(6)
@@ -83,6 +84,7 @@ def rasterize_pdf(
             f'-r{raster_dpi.x:f}x{raster_dpi.y:f}',
         ]
         + (['-dFILTERVECTOR'] if filter_vector else [])
+        + (['-dPDFSTOPONERROR'] if stop_on_error else [])
         + [
             '-o',
             '-',
@@ -161,6 +163,7 @@ def generate_pdfa(
     pdf_version: str = '1.5',
     pdfa_part: str = '2',
     progressbar_class=None,
+    stop_on_error: bool = False,
 ):
     # Ghostscript's compression is all or nothing. We can either force all images
     # to JPEG, force all to Flate/PNG, or let it decide how to encode the images.
@@ -193,6 +196,10 @@ def generate_pdfa(
         # https://bugs.ghostscript.com/show_bug.cgi?id=705187
         compression_args.append('-dNEWPDF=false')
 
+    if os.name == 'nt':
+        # Windows has lots of fatal "permission denied" errors
+        stop_on_error = False
+
     # nb no need to specify ProcessColorModel when ColorConversionStrategy
     # is set; see:
     # https://bugs.ghostscript.com/show_bug.cgi?id=699392
@@ -207,6 +214,7 @@ def generate_pdfa(
             "-dAutoRotatePages=/None",
             "-sColorConversionStrategy=" + strategy,
         ]
+        + (['-dPDFSTOPONERROR'] if stop_on_error else [])
         + compression_args
         + [
             "-dJPEGQ=95",

src/ocrmypdf/_pipeline.py

@@ -343,6 +343,7 @@ def rasterize_preview(input_file: Path, page_context: PageContext) -> Path:
         page_dpi=page_dpi,
         rotation=0,
         filter_vector=False,
+        stop_on_soft_error=not page_context.options.continue_on_soft_render_error,
     )
     return output_file
 
@@ -455,6 +456,7 @@ def rasterize(
         pageno=pageinfo.pageno + 1,
         rotation=correction,
         filter_vector=remove_vectors,
+        stop_on_soft_error=not page_context.options.continue_on_soft_render_error,
     )
     return output_file
 
@@ -736,6 +738,7 @@ def convert_to_pdfa(input_pdf: Path, input_ps_stub: Path, context: PdfContext) -
             if options.progress_bar
             else None
         ),
+        stop_on_soft_error=not options.continue_on_soft_render_error,
     )
 
     return output_file

src/ocrmypdf/api.py

@@ -248,6 +248,7 @@ def ocr(  # noqa: ruff: disable=D417
     user_words: os.PathLike | None = None,
     user_patterns: os.PathLike | None = None,
     fast_web_view: float | None = None,
+    continue_on_soft_render_error: bool | None = None,
     plugins: Iterable[StrPath] | None = None,
     plugin_manager=None,
     keep_temporary_files: bool | None = None,

src/ocrmypdf/builtin_plugins/ghostscript.py

@@ -45,6 +45,7 @@ def rasterize_pdf_page(
     page_dpi,
     rotation,
     filter_vector,
+    stop_on_soft_error,
 ):
     """Rasterize a single page of a PDF file using Ghostscript."""
     ghostscript.rasterize_pdf(
@@ -56,6 +57,7 @@ def rasterize_pdf_page(
         page_dpi=page_dpi,
         rotation=rotation,
         filter_vector=filter_vector,
+        stop_on_error=stop_on_soft_error,
     )
     return output_file
 
@@ -69,6 +71,7 @@ def generate_pdfa(
     pdf_version,
     pdfa_part,
     progressbar_class,
+    stop_on_soft_error,
 ):
     """Generate a PDF/A from the list of PDF pages and PDF/A metadata."""
     ghostscript.generate_pdfa(
@@ -78,5 +81,6 @@ def generate_pdfa(
         pdf_version=pdf_version,
         pdfa_part=pdfa_part,
         progressbar_class=progressbar_class,
+        stop_on_error=stop_on_soft_error,
     )
     return output_file

src/ocrmypdf/cli.py

@@ -420,6 +420,14 @@ Online documentation is located at:
         "which do not benefit. If the threshold is 0 it will be apply to all files. "
         "Set the threshold very high to disable.",
     )
+    advanced.add_argument(
+        '--continue-on-soft-render-error',
+        action='store_true',
+        help="Continue processing pages after a recoverable PDF rendering error. "
+        "A recoverable error is one that does not prevent the page from being "
+        "rendered, but may result in visual differences compared to the input "
+        "file. Missing fonts are a typical source of these errors.",
+    )
     advanced.add_argument(
         '--plugin',
         dest='plugins',

src/ocrmypdf/pluginspec.py

@@ -203,6 +203,7 @@ def rasterize_pdf_page(
     page_dpi: Resolution | None,
     rotation: int | None,
     filter_vector: bool,
+    stop_on_soft_error: bool,
 ) -> Path:
     """Rasterize one page of a PDF at resolution raster_dpi in canvas units.
 
@@ -213,19 +214,26 @@ def rasterize_pdf_page(
     Args:
         input_file: The PDF to rasterize.
         output_file: The desired name of the rasterized image.
-        raster_device: Type of image to produce at output_file
-        raster_dpi: Resolution at which to rasterize page
-        pageno: Page number to rasterize (beginning at page 1)
-        page_dpi: Resolution, overriding output image DPI
-        rotation: Cardinal angle, clockwise, to rotate page
-        filter_vector: If True, remove vector graphics objects
+        raster_device: Type of image to produce at output_file.
+        raster_dpi: Resolution in dots per inch at which to rasterize page.
+        pageno: Page number to rasterize (beginning at page 1).
+        page_dpi: Resolution, overriding output image DPI.
+        rotation: Cardinal angle, clockwise, to rotate page.
+        filter_vector: If True, remove vector graphics objects.
+        stop_on_soft_error: If there is an "soft error" such that PDF page image
+            generation can proceed, but may visually differ from the original,
+            the implementer of this hook should raise a detailed exception. If
+            ``False``, continue processing and report by logging it. If the hook
+            cannot proceed, it should always raise an exception, regardless of
+            this setting. One "soft error" would be a missing font that is
+            required to properly rasterize the PDF.
+
     Returns:
         Path: output_file if successful
     Note:
         This hook will be called from child processes. Modifying global state
         will not affect the main process or other child processes.
 
-
     Note:
         This is a :ref:`firstresult hook<firstresult>`.
     """
@@ -462,6 +470,7 @@ def generate_pdfa(
     pdf_version: str,
     pdfa_part: str,
     progressbar_class,
+    stop_on_soft_error: bool,
 ) -> Path:
     """Generate a PDF/A.
 
@@ -492,6 +501,12 @@ def generate_pdfa(
             and the name of the work units ("page"). Then ``instance.update()``
             will be called when a work unit is completed. If ``None``, no
             progress information is reported.
+        stop_on_soft_error: If there is an "soft error" such that PDF/A generation
+            can proceed and produce a valid PDF/A, but output may be invalid or
+            may not visually resemble the original, the implementer of this hook
+            should raise a detailed exception. If ``False``, continue processing
+            and report by logging it. If the hook cannot proceed, it should always
+            raise an exception, regardless of this setting.
 
     Returns:
         Path: If successful, the hook should return ``output_file``.

src/ocrmypdf/subprocess/__init__.py

@@ -118,8 +118,6 @@ def run_polling_stderr(
 def _fix_process_args(
     args: Args, env: OsEnviron | None, kwargs
 ) -> tuple[Args, OsEnviron, logging.Logger, bool]:
-    assert 'universal_newlines' not in kwargs, "Use text= instead of universal_newlines"
-
     if not env:
         env = os.environ
 

tests/plugins/gs_feature_elision.py

@@ -31,6 +31,7 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
             progressbar_class=None,
+            stop_on_soft_error=True,
         )
         mock.assert_called_once()
     return output_file

tests/plugins/gs_pdfa_failure.py

@@ -33,6 +33,7 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
             progressbar_class=None,
+            stop_on_soft_error=True,
         )
         mock.assert_called()
         return output_file

tests/plugins/gs_raster_failure.py

@@ -39,6 +39,7 @@ def rasterize_pdf_page(
             page_dpi=page_dpi,
             rotation=rotation,
             filter_vector=filter_vector,
+            stop_on_soft_error=True,
         )
         mock.assert_called()
         return output_file

tests/plugins/gs_raster_soft_error.py

@@ -0,0 +1,47 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+from pathlib import Path
+from subprocess import CalledProcessError
+from unittest.mock import patch
+
+from ocrmypdf import hookimpl
+from ocrmypdf.builtin_plugins import ghostscript
+from ocrmypdf.subprocess import run
+
+
+def fail_if_stoponerror(args, **kwargs):
+    if '-dPDFSTOPONERROR' in args:
+        raise CalledProcessError(1, 'gs', output=b"", stderr=b"PDF STOP ON ERROR")
+    return run(args, **kwargs)
+
+
+@hookimpl
+def rasterize_pdf_page(
+    input_file,
+    output_file,
+    raster_device,
+    raster_dpi,
+    pageno,
+    page_dpi,
+    rotation,
+    filter_vector,
+    stop_on_soft_error,
+) -> Path:
+    with patch('ocrmypdf._exec.ghostscript.run') as mock:
+        mock.side_effect = fail_if_stoponerror
+        ghostscript.rasterize_pdf_page(
+            input_file=input_file,
+            output_file=output_file,
+            raster_device=raster_device,
+            raster_dpi=raster_dpi,
+            pageno=pageno,
+            page_dpi=page_dpi,
+            rotation=rotation,
+            filter_vector=filter_vector,
+            stop_on_soft_error=stop_on_soft_error,
+        )
+        mock.assert_called()
+        return output_file

tests/plugins/gs_render_failure.py

@@ -28,6 +28,7 @@ def generate_pdfa(pdf_pages, pdfmark, output_file, compression, pdf_version, pdf
             pdf_version=pdf_version,
             pdfa_part=pdfa_part,
             progressbar_class=None,
+            stop_on_soft_error=True,
         )
         mock.assert_called()
         return output_file

tests/plugins/gs_render_soft_error.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2022 James R. Barlow
+# SPDX-License-Identifier: MIT
+
+from __future__ import annotations
+
+from pathlib import Path
+from subprocess import CalledProcessError
+from unittest.mock import patch
+
+from ocrmypdf import hookimpl
+from ocrmypdf.builtin_plugins import ghostscript
+from ocrmypdf.subprocess import run_polling_stderr
+
+
+def fail_if_stoponerror(args, **kwargs):
+    if '-dPDFSTOPONERROR' in args:
+        raise CalledProcessError(1, 'gs', output=b"", stderr=b"PDF STOP ON ERROR")
+    return run_polling_stderr(args, **kwargs)
+
+
+@hookimpl
+def generate_pdfa(
+    pdf_pages,
+    pdfmark,
+    output_file,
+    compression,
+    pdf_version,
+    pdfa_part,
+    stop_on_soft_error,
+):
+    with patch('ocrmypdf._exec.ghostscript.run_polling_stderr') as mock:
+        mock.side_effect = fail_if_stoponerror
+        ghostscript.generate_pdfa(
+            pdf_pages=pdf_pages,
+            pdfmark=pdfmark,
+            output_file=output_file,
+            compression=compression,
+            pdf_version=pdf_version,
+            pdfa_part=pdfa_part,
+            progressbar_class=None,
+            stop_on_soft_error=stop_on_soft_error,
+        )
+        mock.assert_called()
+        return output_file

tests/test_helpers.py

@@ -17,6 +17,7 @@ from ocrmypdf import helpers
 from .conftest import running_in_docker
 
 needs_symlink = pytest.mark.skipif(os.name == 'nt', reason='needs posix symlink')
+windows_only = pytest.mark.skipif(os.name != 'nt', reason="Windows test")
 
 
 class TestSafeSymlink:
@@ -93,7 +94,7 @@ class TestFileIsWritable:
         assert not helpers.is_file_writable(pathmock)
 
 
-@pytest.mark.skipif(os.name != 'nt', reason="Windows test")
+@windows_only
 def test_gs_install_locations():
     # pylint: disable=import-outside-toplevel
     from ocrmypdf.subprocess._windows import _gs_version_in_path_key
@@ -104,7 +105,7 @@ def test_gs_install_locations():
     )
 
 
-@pytest.mark.skipif(os.name != 'nt', reason="Windows test")
+@windows_only
 def test_shim_paths(tmp_path):
     # pylint: disable=import-outside-toplevel
     from ocrmypdf.subprocess._windows import shim_env_path

tests/test_metadata.py

@@ -350,7 +350,15 @@ def test_malformed_docinfo(caplog, resources, outdir):
         pike.save(outdir / 'layers.rendered.pdf', fix_metadata_version=False)
 
     options = get_parser().parse_args(
-        args=['-j', '1', '--output-type', 'pdfa-2', 'a.pdf', 'b.pdf']
+        args=[
+            '-j',
+            '1',
+            '--continue-on-soft-render-error',
+            '--output-type',
+            'pdfa-2',
+            'a.pdf',
+            'b.pdf',
+        ]
     )
     pdfinfo = PdfInfo(outdir / 'layers.rendered.pdf')
     context = PdfContext(

tests/test_rotation.py

@@ -275,6 +275,7 @@ def test_rasterize_rotates(resources, tmp_path):
         pageno=1,
         rotation=90,
         filter_vector=False,
+        stop_on_soft_error=True,
     )
     with Image.open(img) as im:
         assert im.size == (83, 200), "Image not rotated"
@@ -289,6 +290,7 @@ def test_rasterize_rotates(resources, tmp_path):
         pageno=1,
         rotation=180,
         filter_vector=False,
+        stop_on_soft_error=True,
     )
     assert Image.open(img).size == (200, 83), "Image not rotated"
 

tests/test_soft_error.py

@@ -0,0 +1,63 @@
+# SPDX-FileCopyrightText: 2023 James R. Barlow
+# SPDX-License-Identifier: MPL-2.0
+
+from __future__ import annotations
+
+import os
+
+import pytest
+
+from ocrmypdf.exceptions import ExitCode
+
+from .conftest import run_ocrmypdf
+
+
+def test_raster_continue_on_soft_error(resources, outpdf):
+    p = run_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--continue-on-soft-render-error',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--plugin',
+        'tests/plugins/gs_raster_soft_error.py',
+    )
+    assert p.returncode == ExitCode.ok
+
+
+def test_raster_stop_on_soft_error(resources, outpdf):
+    p = run_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--plugin',
+        'tests/plugins/gs_raster_soft_error.py',
+    )
+    assert p.returncode == ExitCode.child_process_error
+
+
+def test_render_continue_on_soft_error(resources, outpdf):
+    p = run_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--continue-on-soft-render-error',
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--plugin',
+        'tests/plugins/gs_render_soft_error.py',
+    )
+    assert p.returncode == ExitCode.ok
+
+
+@pytest.mark.skipif(os.name == 'nt', reason='Ghostscript on Windows errors out')
+def test_render_stop_on_soft_error(resources, outpdf):
+    p = run_ocrmypdf(
+        resources / 'francais.pdf',
+        outpdf,
+        '--plugin',
+        'tests/plugins/tesseract_noop.py',
+        '--plugin',
+        'tests/plugins/gs_render_soft_error.py',
+    )
+    assert p.returncode == ExitCode.child_process_error