Add PDF linearization

docs/release_notes.rst

@@ -23,33 +23,35 @@ v9.0.0
    implements this feature (Leptonica).
 -  The ``-v`` (verbosity level) parameter now accepts only ``0``, ``1``, and
    ``2``.
--  Dropped support for Tesseract "4.00.00-alpha" releases. Tesseract 4.0 beta and
+-  Dropped support for Tesseract 4.00.00-alpha releases. Tesseract 4.0 beta and
    later remain supported.
+-  Dropped the ``ocrmypdf-polyglot`` and ``ocrmypdf-webservice`` images.
 
-**Major changes**
+**New features**
 
 -  Added a high level API for applications that want to integrate OCRmyPDF.
    Special thanks to Martin Wind (@mawi1988) whose made significant contributions
    to this effort. OCRmyPDF is GPLv3-licensed.
--  Major internal code reorganization.
 -  Added progress bars for long-running steps. ■■■■■■■□□
+-  We now create linearized ("fast web view") PDFs by default. The new parameter
+   ``--fast-web-view`` provides control over when this feature is applied.
+-  Added a new ``--pages`` feature to limit OCR to only a specific page range.
+   The list may contain commas or single pages, such as ``1, 3, 5-11``.
 -  When the number of pages is small compared to the number of allowed jobs, we
    run Tesseract in multithreaded (OpenMP) mode when available. This should
    improve performance on files with low page counts.
--  Pages with vector artwork are treated as full color. Previously, vectors
-   were ignored when considering the colorspace needed to cover a page, which
-   could cause loss of color under certain settings.
--  Dropped the ``ocrmypdf-polyglot`` and ``ocrmypdf-webservice`` images.
 -  Removed dependency on ``ruffus``, and with that, the non-reentrancy
    restrictions that previous made an API impossible.
--  Added a new ``--pages`` feature to limit OCR to only a specific page range.
-   The list may contain commas or single pages, such as ``1, 3, 5-11``.
 -  Output and logging messages overhauled so that ocrmypdf may be integrated
    into applications that use the logging module.
 -  pikepdf 1.6.0 is required.
+-  Added a logo. 😊
 
-**Minor changes**
+**Bug fixes**
 
+-  Pages with vector artwork are treated as full color. Previously, vectors
+   were ignored when considering the colorspace needed to cover a page, which
+   could cause loss of color under certain settings.
 -  Test suite now spawns processes less frequently, allowing more accurate
    measurement of code coverage.
 -  Improved test coverage.
@@ -57,7 +59,6 @@ v9.0.0
 -  Updated Docker images to use newer versions.
 -  Fixed images encoded as JBIG2 with a colorspace other than ``/DeviceGray``
    were not interpreted correctly.
--  We have a logo. 😊
 
 v8.3.2
 ======

src/ocrmypdf/_pipeline.py

@@ -716,6 +716,13 @@ def convert_to_pdfa(input_pdf, input_ps_stub, context):
     return output_file
 
 
+def should_linearize(working_file, context):
+    filesize = os.stat(working_file).st_size
+    if filesize > (context.options.fast_web_view * 1_000_000):
+        return True
+    return False
+
+
 def metadata_fixup(working_file, context):
     output_file = context.get_path('metafix.pdf')
     options = context.options
@@ -749,11 +756,14 @@ def metadata_fixup(working_file, context):
                 context.log.info(
                     "The following metadata fields were not copied: %r", not_copied
                 )
-
     pdf.save(
         output_file,
         compress_streams=True,
+        preserve_pdfa=True,
         object_stream_mode=pikepdf.ObjectStreamMode.generate,
+        linearize=(  # Don't linearize if optimize() will be linearizing too
+            should_linearize(working_file, context) if options.optimize == 0 else False
+        ),
     )
     original.close()
     pdf.close()
@@ -762,7 +772,13 @@ def metadata_fixup(working_file, context):
 
 def optimize_pdf(input_file, context):
     output_file = context.get_path('optimize.pdf')
-    optimize(input_file, output_file, context)
+    save_settings = dict(
+        compress_streams=True,
+        preserve_pdfa=True,
+        object_stream_mode=pikepdf.ObjectStreamMode.generate,
+        linearize=should_linearize(input_file, context),
+    )
+    optimize(input_file, output_file, context, save_settings)
     return output_file
 
 

src/ocrmypdf/api.py

@@ -196,6 +196,7 @@ def ocr(  # pylint: disable=unused-argument
     pdfa_image_compression=None,
     user_words=None,
     user_patterns=None,
+    fast_web_view=None,
     keep_temporary_files=None,
     progress_bar=None,
     tesseract_env=None,

src/ocrmypdf/cli.py

@@ -466,6 +466,18 @@ advanced.add_argument(
     metavar='FILE',
     help="Specify the location of the Tesseract user patterns file.",
 )
+advanced.add_argument(
+    '--fast-web-view',
+    type=numeric(float, 0),
+    default=1.0,
+    metavar="MEGABYTES",
+    help="If the size of file is more than this threshold (in MB), then "
+    "linearize the PDF for fast web viewing. This allows the PDF to be "
+    "displayed before it is fully downloaded in web browsers, but increases "
+    "the space required slightly. By default we skip this for small files "
+    "which do not benefit. If the threshold is 0 it will be apply to all files. "
+    "Set the threshold very high to disable.",
+)
 
 debugging = parser.add_argument_group(
     "Debugging", "Arguments to help with troubleshooting and debugging"

src/ocrmypdf/optimize.py

@@ -449,7 +449,7 @@ def transcode_pngs(pike, images, image_name_fn, root, log, options):
         im_obj.write(compdata.read(), filter=Name.FlateDecode, decode_parms=dparms)
 
 
-def optimize(input_file, output_file, context):
+def optimize(input_file, output_file, context, save_settings):
     log = context.log
     options = context.options
     if options.optimize == 0:
@@ -479,11 +479,7 @@ def optimize(input_file, output_file, context):
 
         target_file = Path(output_file).with_suffix('.opt.pdf')
         pike.remove_unreferenced_resources()
-        pike.save(
-            target_file,
-            preserve_pdfa=True,
-            object_stream_mode=pikepdf.ObjectStreamMode.generate,
-        )
+        pike.save(target_file, **save_settings)
 
     input_size = Path(input_file).stat().st_size
     output_size = Path(target_file).stat().st_size
@@ -497,8 +493,11 @@ def optimize(input_file, output_file, context):
     log.info(f"Optimize ratio: {ratio:.2f} savings: {(100 * savings):.1f}%")
 
     if savings < 0:
-        log.info("Optimize did not improve the file - discarded")
-        re_symlink(input_file, output_file)
+        log.info("Image optimization did not improve the file - discarded")
+        # We still need to save the file
+        with pikepdf.open(input_file) as pike:
+            pike.remove_unreferenced_resources()
+            pike.save(output_file, **save_settings)
     else:
         re_symlink(target_file, output_file)
 
@@ -535,7 +534,16 @@ def main(infile, outfile, level, jobs=1):
     with TemporaryDirectory() as td:
         context = PDFContext(options, td, infile, None)
         tmpout = Path(td) / 'out.pdf'
-        optimize(infile, tmpout, context)
+        optimize(
+            infile,
+            tmpout,
+            context,
+            dict(
+                compress_streams=True,
+                preserve_pdfa=True,
+                object_stream_mode=pikepdf.ObjectStreamMode.generate,
+            ),
+        )
         copy(fspath(tmpout), fspath(outfile))
 
 

tests/test_main.py

@@ -28,6 +28,7 @@ import pytest
 from PIL import Image
 
 import ocrmypdf
+import pikepdf
 from ocrmypdf.exceptions import ExitCode, MissingDependencyError
 from ocrmypdf.exec import ghostscript, qpdf, tesseract
 from ocrmypdf.leptonica import Pix
@@ -1093,3 +1094,34 @@ def test_version_check():
 
     with pytest.raises(MissingDependencyError):
         get_version('echo')
+
+
+@pytest.mark.parametrize(
+    'threshold, optimize, output_type, expected',
+    [
+        [1.0, 0, 'pdfa', False],
+        [1.0, 0, 'pdf', False],
+        [0.0, 0, 'pdfa', True],
+        [0.0, 0, 'pdf', True],
+        [1.0, 1, 'pdfa', False],
+        [1.0, 1, 'pdf', False],
+        [0.0, 1, 'pdfa', True],
+        [0.0, 1, 'pdf', True],
+    ],
+)
+def test_fast_web_view(
+    spoof_tesseract_noop, resources, outpdf, threshold, optimize, output_type, expected
+):
+    check_ocrmypdf(
+        resources / 'trivial.pdf',
+        outpdf,
+        '--fast-web-view',
+        threshold,
+        '--optimize',
+        optimize,
+        '--output-type',
+        output_type,
+        env=spoof_tesseract_noop,
+    )
+    with pikepdf.open(outpdf) as pdf:
+        assert pdf.is_linearized == expected