diff --git a/src/ocrmypdf/__main__.py b/src/ocrmypdf/__main__.py index 8955252c..3cae0405 100755 --- a/src/ocrmypdf/__main__.py +++ b/src/ocrmypdf/__main__.py @@ -249,12 +249,25 @@ ocrsettings.add_argument( help="Skip OCR on any pages that already contain text, but include the " "page in final output; useful for PDFs that contain a mix of " "images, text pages, and/or previously OCRed pages") - ocrsettings.add_argument( '--skip-big', type=float, metavar='MPixels', help="Skip OCR on pages larger than the specified amount of megapixels, " "but include skipped pages in final output") +optimizing = parser.add_argument_group( + "Optimization options", + "Control how the PDF is optimized after OCR" +) +optimizing.add_argument( + '-O', '--optimize', type=int, choices=range(0, 4), default=1, + help=("Control how PDF is optimized after processing:" + "0 - do not optimize;" + "1 - do safe, lossless optimizations (default);" + "2 - do lossy optimizations; " + "3 - do aggressive lossy optimizations" + ) +) + advanced = parser.add_argument_group( "Advanced", "Advanced options to control Tesseract's OCR behavior") @@ -431,18 +444,24 @@ def check_options_sidecar(options, log): options.sidecar = options.output_file + '.txt' +def _optional_program_check(name, version_fn, min_version, for_argument): + try: + if version_fn() < min_version: + raise MissingDependencyError( + "The installed '{}' is not supported. " + "Install version {} or newer.".format(name, min_version)) + except FileNotFoundError: + raise MissingDependencyError( + "Install the '{}' program to use {}.".format(name, for_argument)) + + def check_options_preprocessing(options, log): if any((options.clean, options.clean_final)): from .exec import unpaper - try: - if unpaper.version() < '6.1': - raise MissingDependencyError( - "The installed 'unpaper' is not supported. " - "Install version 6.1 or newer.") - except FileNotFoundError: - raise MissingDependencyError( - "Install the 'unpaper' program to use --clean, --clean-final.") - + _optional_program_check( + 'unpaper', unpaper.version, '6.1', '--clean, --clean-final' + ) + def check_options_ocr_behavior(options, log): if options.force_ocr and options.skip_text: @@ -451,6 +470,14 @@ def check_options_ocr_behavior(options, log): "Error: --force-ocr and --skip-text are mutually incompatible.") +def check_options_optimizing(options, log): + if options.optimize >= 2: + from .exec import pngquant + _optional_program_check( + 'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}' + ) + + def check_options_advanced(options, log): if options.tesseract_oem and not tesseract.v4(): log.warning( @@ -491,6 +518,7 @@ def check_options(options, log): check_options_sidecar(options, log) check_options_preprocessing(options, log) check_options_ocr_behavior(options, log) + check_options_optimizing(options, log) check_options_advanced(options, log) check_options_pillow(options, log) except ValueError as e: diff --git a/src/ocrmypdf/_optimize.py b/src/ocrmypdf/_optimize.py index 9f084cfe..fffdc28a 100644 --- a/src/ocrmypdf/_optimize.py +++ b/src/ocrmypdf/_optimize.py @@ -16,7 +16,7 @@ # along with OCRmyPDF. If not, see . from pathlib import Path -from subprocess import run, PIPE +from subprocess import CalledProcessError import concurrent.futures from collections import defaultdict import struct @@ -28,7 +28,7 @@ import pikepdf from . import leptonica from .helpers import re_symlink -from .exec import pngquant +from .exec import pngquant, jbig2enc PAGE_GROUP_SIZE = 10 SIMPLE_COLORSPACES = ('/DeviceRGB', '/DeviceGray', '/CalRGB', '/CalGray') @@ -115,8 +115,8 @@ def make_img_name(root, xref): return str(root / '{:08d}.png'.format(xref)) -def extract_image(doc, pike, root, log, image, xref, jbig2s, - pngs, jpegs): +def extract_image(*, doc, pike, root, log, image, xref, jbig2s, + pngs, jpegs, options): if image.Subtype != '/Image': return False if image.Length < 100: @@ -152,7 +152,9 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s, jbig2s.append(xref) elif filtdp[0] == '/JPXDecode': return False - elif filtdp[0] == '/DCTDecode' and cs in SIMPLE_COLORSPACES: + elif filtdp[0] == '/DCTDecode' \ + and cs in SIMPLE_COLORSPACES \ + and options.optimize >= 2: raw_jpeg = pike._get_object_id(xref, 0) color_transform = filtdp[1].get('/ColorTransform', 1) if color_transform != 1: @@ -170,8 +172,8 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s, raw_jpeg_data = raw_jpeg.read_raw_bytes() (root / '{:08d}.jpg'.format(xref)).write_bytes(raw_jpeg_data) jpegs.append(xref) - elif cs in SIMPLE_COLORSPACES: - # For any 'inferior' filter include /FlateDecode we extract + elif cs in SIMPLE_COLORSPACES and fitz: + # For any 'inferior' filter including /FlateDecode we extract # and recode as /FlateDecode # raw_png = pike._get_object_id(xref, 0) # raw_png_data = raw_png.read_raw_bytes() @@ -185,7 +187,7 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s, return True -def extract_images(doc, pike, root, log): +def extract_images(doc, pike, root, log, options): # Extract images we can improve changed_xrefs = set() jbig2_groups = defaultdict(lambda: []) @@ -204,8 +206,10 @@ def extract_images(doc, pike, root, log): continue # Don't improve same image twice try: result = extract_image( - doc, pike, root, log, image, xref, - jbig2_groups[group], pngs, jpegs) + doc=doc, pike=pike, root=root, log=log, image=image, + xref=xref, jbig2s=jbig2_groups[group], pngs=pngs, + jpegs=jpegs, options=options + ) if result: changed_xrefs.add(xref) except Exception as e: @@ -244,14 +248,15 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options): futures = [] for group, xrefs in jbig2_groups.items(): prefix = 'group{:08d}'.format(group) - cmd = ['jbig2', '-b', prefix, '-s', '-p'] - cmd.extend(make_img_name(root, xref) for xref in xrefs) future = executor.submit( - run, cmd, cwd=str(root), stdout=PIPE, stderr=PIPE) + jbig2enc.convert_group, + cwd=str(root), + infiles=(make_img_name(root, xref) for xref in xrefs), + out_prefix=prefix + ) futures.append(future) for future in concurrent.futures.as_completed(futures): proc = future.result() - proc.check_returncode() log.debug(proc.stderr) for group, xrefs in jbig2_groups.items(): @@ -292,12 +297,14 @@ def transcode_jpegs(pike, jpegs, root, log, options): def transcode_pngs(pike, pngs, root, options): - with concurrent.futures.ThreadPoolExecutor( - max_workers=options.jobs) as executor: - for xref in pngs: - executor.submit( - pngquant.quantize, - make_img_name(root, xref), make_img_name(root, xref), 65, 80) + if options.optimize >= 2: + with concurrent.futures.ThreadPoolExecutor( + max_workers=options.jobs) as executor: + for xref in pngs: + executor.submit( + pngquant.quantize, + make_img_name(root, xref), make_img_name(root, xref), + 65, 80) for xref in pngs: im_obj = pike._get_object_id(xref, 0) @@ -335,18 +342,21 @@ def optimize( log, context): - if not fitz: - re_symlink(input_file, output_file) + options = context.get_options() + if options.optimize == 0: + re_symlink(input_file, output_file, log) return - options = context.get_options() - doc = fitz.open(input_file) + if fitz: + doc = fitz.open(input_file) + else: + doc = None pike = pikepdf.Pdf.open(input_file) root = Path(output_file).parent / 'images' root.mkdir(exist_ok=True) changed_xrefs, jbig2_groups, jpegs, pngs = extract_images( - doc, pike, root, log) + doc, pike, root, log, options) convert_to_jbig2(pike, jbig2_groups, root, log, options) @@ -367,9 +377,9 @@ def optimize( if savings < 0: log.info("Optimize did not improve the file - discarded") - re_symlink(input_file, output_file) + re_symlink(input_file, output_file, log) else: - re_symlink(target_file, output_file) + re_symlink(target_file, output_file, log) if __name__ == '__main__': diff --git a/src/ocrmypdf/exec/jbig2enc.py b/src/ocrmypdf/exec/jbig2enc.py new file mode 100644 index 00000000..10d1285c --- /dev/null +++ b/src/ocrmypdf/exec/jbig2enc.py @@ -0,0 +1,44 @@ +# © 2018 James R. Barlow: github.com/jbarlow83 +# +# This file is part of OCRmyPDF. +# +# OCRmyPDF is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# OCRmyPDF is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with OCRmyPDF. If not, see . + +from subprocess import CalledProcessError, run, PIPE +from functools import lru_cache +import sys +import os +import shutil + +from . import get_version +from ..exceptions import ExitCode + + +@lru_cache(maxsize=1) +def version(): + return get_version('jbig2enc', regex=r'jbig2enc (\d+(\.\d+)*).*') + + +def convert_group(*, cwd, infiles, out_prefix): + args = [ + 'jbig2', + '-b', + out_prefix, + '-s', + '-p', + ] + args.extend(infiles) + proc = run(args, cwd=cwd, stdout=PIPE, stderr=PIPE) + proc.check_returncode() + return proc \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index ab035ab3..18477bc8 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -715,6 +715,7 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec, # Runs: ocrmypdf - output.pdf < testfile with open(input_file, 'rb') as input_stream: p_args = ocrmypdf_exec + [ + '--optimize', '0', '--image-dpi', '150', '--output-type', 'pdf', '-', output_file] p = Popen( p_args, close_fds=True, stdout=PIPE, stderr=PIPE, @@ -757,6 +758,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec, with open(input_file, 'rb') as input_stream: p_args = ocrmypdf_exec + [ '--image-dpi', '150', '--output-type', 'pdfa', + '--optimize', '0', '--pdfa-image-compression', compression, '-', output_file] p = Popen(