menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add arguments to control optimization

Browse Source
This commit is contained in:
James R. Barlow
2018-05-10 22:23:24 -07:00
parent 40d09ddb23
commit 72253d09fa
4 changed files with 121 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ocrmypdf/__main__.py
+38 -10
View File
@@ -249,12 +249,25 @@ ocrsettings.add_argument(
help="Skip OCR on any pages that already contain text, but include the "
"page in final output; useful for PDFs that contain a mix of "
"images, text pages, and/or previously OCRed pages")
ocrsettings.add_argument(
'--skip-big', type=float, metavar='MPixels',
help="Skip OCR on pages larger than the specified amount of megapixels, "
"but include skipped pages in final output")
optimizing = parser.add_argument_group(
"Optimization options",
"Control how the PDF is optimized after OCR"
)
optimizing.add_argument(
'-O', '--optimize', type=int, choices=range(0, 4), default=1,
help=("Control how PDF is optimized after processing:"
"0 - do not optimize;"
"1 - do safe, lossless optimizations (default);"
"2 - do lossy optimizations; "
"3 - do aggressive lossy optimizations"
)
)
advanced = parser.add_argument_group(
"Advanced",
"Advanced options to control Tesseract's OCR behavior")
@@ -431,18 +444,24 @@ def check_options_sidecar(options, log):
options.sidecar = options.output_file + '.txt'
def _optional_program_check(name, version_fn, min_version, for_argument):
try:
if version_fn() < min_version:
raise MissingDependencyError(
"The installed '{}' is not supported. "
"Install version {} or newer.".format(name, min_version))
except FileNotFoundError:
raise MissingDependencyError(
"Install the '{}' program to use {}.".format(name, for_argument))
def check_options_preprocessing(options, log):
if any((options.clean, options.clean_final)):
from .exec import unpaper
try:
if unpaper.version() < '6.1':
raise MissingDependencyError(
"The installed 'unpaper' is not supported. "
"Install version 6.1 or newer.")
except FileNotFoundError:
raise MissingDependencyError(
"Install the 'unpaper' program to use --clean, --clean-final.")
_optional_program_check(
'unpaper', unpaper.version, '6.1', '--clean, --clean-final'
)
def check_options_ocr_behavior(options, log):
if options.force_ocr and options.skip_text:
@@ -451,6 +470,14 @@ def check_options_ocr_behavior(options, log):
"Error: --force-ocr and --skip-text are mutually incompatible.")
def check_options_optimizing(options, log):
if options.optimize >= 2:
from .exec import pngquant
_optional_program_check(
'pngquant', pngquant.version, '2.0.1', '--optimize {2,3}'
)
def check_options_advanced(options, log):
if options.tesseract_oem and not tesseract.v4():
log.warning(
@@ -491,6 +518,7 @@ def check_options(options, log):
check_options_sidecar(options, log)
check_options_preprocessing(options, log)
check_options_ocr_behavior(options, log)
check_options_optimizing(options, log)
check_options_advanced(options, log)
check_options_pillow(options, log)
except ValueError as e:
src/ocrmypdf/_optimize.py
+37 -27
View File
@@ -16,7 +16,7 @@
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
from pathlib import Path
from subprocess import run, PIPE
from subprocess import CalledProcessError
import concurrent.futures
from collections import defaultdict
import struct
@@ -28,7 +28,7 @@ import pikepdf
from . import leptonica
from .helpers import re_symlink
from .exec import pngquant
from .exec import pngquant, jbig2enc
PAGE_GROUP_SIZE = 10
SIMPLE_COLORSPACES = ('/DeviceRGB', '/DeviceGray', '/CalRGB', '/CalGray')
@@ -115,8 +115,8 @@ def make_img_name(root, xref):
return str(root / '{:08d}.png'.format(xref))
def extract_image(doc, pike, root, log, image, xref, jbig2s,
pngs, jpegs):
def extract_image(*, doc, pike, root, log, image, xref, jbig2s,
pngs, jpegs, options):
if image.Subtype != '/Image':
return False
if image.Length < 100:
@@ -152,7 +152,9 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s,
jbig2s.append(xref)
elif filtdp[0] == '/JPXDecode':
return False
elif filtdp[0] == '/DCTDecode' and cs in SIMPLE_COLORSPACES:
elif filtdp[0] == '/DCTDecode' \
and cs in SIMPLE_COLORSPACES \
and options.optimize >= 2:
raw_jpeg = pike._get_object_id(xref, 0)
color_transform = filtdp[1].get('/ColorTransform', 1)
if color_transform != 1:
@@ -170,8 +172,8 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s,
raw_jpeg_data = raw_jpeg.read_raw_bytes()
(root / '{:08d}.jpg'.format(xref)).write_bytes(raw_jpeg_data)
jpegs.append(xref)
elif cs in SIMPLE_COLORSPACES:
# For any 'inferior' filter include /FlateDecode we extract
elif cs in SIMPLE_COLORSPACES and fitz:
# For any 'inferior' filter including /FlateDecode we extract
# and recode as /FlateDecode
# raw_png = pike._get_object_id(xref, 0)
# raw_png_data = raw_png.read_raw_bytes()
@@ -185,7 +187,7 @@ def extract_image(doc, pike, root, log, image, xref, jbig2s,
return True
def extract_images(doc, pike, root, log):
def extract_images(doc, pike, root, log, options):
# Extract images we can improve
changed_xrefs = set()
jbig2_groups = defaultdict(lambda: [])
@@ -204,8 +206,10 @@ def extract_images(doc, pike, root, log):
continue # Don't improve same image twice
try:
result = extract_image(
doc, pike, root, log, image, xref,
jbig2_groups[group], pngs, jpegs)
doc=doc, pike=pike, root=root, log=log, image=image,
xref=xref, jbig2s=jbig2_groups[group], pngs=pngs,
jpegs=jpegs, options=options
)
if result:
changed_xrefs.add(xref)
except Exception as e:
@@ -244,14 +248,15 @@ def convert_to_jbig2(pike, jbig2_groups, root, log, options):
futures = []
for group, xrefs in jbig2_groups.items():
prefix = 'group{:08d}'.format(group)
cmd = ['jbig2', '-b', prefix, '-s', '-p']
cmd.extend(make_img_name(root, xref) for xref in xrefs)
future = executor.submit(
run, cmd, cwd=str(root), stdout=PIPE, stderr=PIPE)
jbig2enc.convert_group,
cwd=str(root),
infiles=(make_img_name(root, xref) for xref in xrefs),
out_prefix=prefix
)
futures.append(future)
for future in concurrent.futures.as_completed(futures):
proc = future.result()
proc.check_returncode()
log.debug(proc.stderr)
for group, xrefs in jbig2_groups.items():
@@ -292,12 +297,14 @@ def transcode_jpegs(pike, jpegs, root, log, options):
def transcode_pngs(pike, pngs, root, options):
with concurrent.futures.ThreadPoolExecutor(
max_workers=options.jobs) as executor:
for xref in pngs:
executor.submit(
pngquant.quantize,
make_img_name(root, xref), make_img_name(root, xref), 65, 80)
if options.optimize >= 2:
with concurrent.futures.ThreadPoolExecutor(
max_workers=options.jobs) as executor:
for xref in pngs:
executor.submit(
pngquant.quantize,
make_img_name(root, xref), make_img_name(root, xref),
65, 80)
for xref in pngs:
im_obj = pike._get_object_id(xref, 0)
@@ -335,18 +342,21 @@ def optimize(
log,
context):
if not fitz:
re_symlink(input_file, output_file)
options = context.get_options()
if options.optimize == 0:
re_symlink(input_file, output_file, log)
return
options = context.get_options()
doc = fitz.open(input_file)
if fitz:
doc = fitz.open(input_file)
else:
doc = None
pike = pikepdf.Pdf.open(input_file)
root = Path(output_file).parent / 'images'
root.mkdir(exist_ok=True)
changed_xrefs, jbig2_groups, jpegs, pngs = extract_images(
doc, pike, root, log)
doc, pike, root, log, options)
convert_to_jbig2(pike, jbig2_groups, root, log, options)
@@ -367,9 +377,9 @@ def optimize(
if savings < 0:
log.info("Optimize did not improve the file - discarded")
re_symlink(input_file, output_file)
re_symlink(input_file, output_file, log)
else:
re_symlink(target_file, output_file)
re_symlink(target_file, output_file, log)
if __name__ == '__main__':
src/ocrmypdf/exec/jbig2enc.py
+44
View File
@@ -0,0 +1,44 @@
# © 2018 James R. Barlow: github.com/jbarlow83
#
# This file is part of OCRmyPDF.
#
# OCRmyPDF is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# OCRmyPDF is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with OCRmyPDF. If not, see <http://www.gnu.org/licenses/>.
from subprocess import CalledProcessError, run, PIPE
from functools import lru_cache
import sys
import os
import shutil
from . import get_version
from ..exceptions import ExitCode
@lru_cache(maxsize=1)
def version():
return get_version('jbig2enc', regex=r'jbig2enc (\d+(\.\d+)*).*')
def convert_group(*, cwd, infiles, out_prefix):
args = [
'jbig2',
'-b',
out_prefix,
'-s',
'-p',
]
args.extend(infiles)
proc = run(args, cwd=cwd, stdout=PIPE, stderr=PIPE)
proc.check_returncode()
return proc
tests/test_main.py
+2
View File
@@ -715,6 +715,7 @@ def test_compression_preserved(spoof_tesseract_noop, ocrmypdf_exec,
# Runs: ocrmypdf - output.pdf < testfile
with open(input_file, 'rb') as input_stream:
p_args = ocrmypdf_exec + [
'--optimize', '0',
'--image-dpi', '150', '--output-type', 'pdf', '-', output_file]
p = Popen(
p_args, close_fds=True, stdout=PIPE, stderr=PIPE,
@@ -757,6 +758,7 @@ def test_compression_changed(spoof_tesseract_noop, ocrmypdf_exec,
with open(input_file, 'rb') as input_stream:
p_args = ocrmypdf_exec + [
'--image-dpi', '150', '--output-type', 'pdfa',
'--optimize', '0',
'--pdfa-image-compression', compression,
'-', output_file]
p = Popen(