menekis/OCRmyPDF
Public Access
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Optimize page splitting by multiprocessing

Browse Source 
Previously page splitting occurred in a single process because it was
not believed to affect performance much. It turned out to be an expensive
operation.

It now scales better with large page sizes although this has a negative
effect on small files.

Overall time changes as follows:

7 page file, 9.02s -> 9.56s
731 page file, 213s -> 97s

WITH --tesseract-timeout 0 --output-type pdf --skip-text

i.e. you don't get a 2.2x speed gain when OCR is available.

Squashed a commit to ix test suite failure on --rotate-pages
Squashed a commit to remove debug code
This commit is contained in:
James R. Barlow
2018-03-23 13:07:51 -07:00
parent 4f1f3b9b51
commit dea8fcfb5b
1 changed files with 52 additions and 12 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ocrmypdf/pipeline.py
+52 -12
View File
@@ -305,7 +305,7 @@ def is_ocr_required(pageinfo, log, options):
return ocr_required
def split_pages(
def pre_split_pages(
input_files,
output_files,
log,
@@ -331,20 +331,45 @@ def split_pages(
pdfinfo = context.get_pdfinfo()
npages = len(pdfinfo)
qpdf.split_pages(input_file, work_folder, npages)
from glob import glob
for filename in glob(os.path.join(work_folder, '*.page.pdf')):
pageinfo = get_pageinfo(filename, context)
# Ruffus needs to see a file for any task it generates, so create
# empty placeholders for every page.
for n in range(npages):
page = Path(work_folder) / '{0:06d}.presplit.pdf'.format(n + 1)
page.touch()
def split_page(
placeholder_file,
output_file,
log,
context):
pageno = page_number(placeholder_file) - 1
input_pdf = context.get_pdfinfo().filename
qpdf.extract_page(input_pdf, output_file, pageno)
def ocr_or_skip(
input_files,
output_files,
log,
context):
options = context.get_options()
work_folder = context.get_work_folder()
pdfinfo = context.get_pdfinfo()
for input_file in input_files:
pageno = page_number(input_file) - 1
pageinfo = pdfinfo[pageno]
alt_suffix = \
'.ocr.page.pdf' if is_ocr_required(pageinfo, log, options) \
else '.skip.page.pdf'
re_symlink(
filename,
input_file,
os.path.join(
work_folder,
os.path.basename(filename)[0:6] + alt_suffix),
os.path.basename(input_file)[0:6] + alt_suffix),
log)
@@ -1035,16 +1060,31 @@ def build_pipeline(options, work_folder, log, context):
extras=[log, context])
# Split (kwargs for split seems to be broken, so pass plain args)
task_split_pages = main_pipeline.split(
split_pages,
task_pre_split_pages = main_pipeline.split(
pre_split_pages,
task_repair_pdf,
os.path.join(work_folder, '*.page.pdf'),
os.path.join(work_folder, '*.presplit.pdf'),
extras=[log, context])
task_split_pages = main_pipeline.transform(
task_func=split_page,
input=task_pre_split_pages,
filter=suffix('.presplit.pdf'),
output='.page.pdf',
output_dir=work_folder,
extras=[log, context])
task_ocr_or_skip = main_pipeline.split(
ocr_or_skip,
task_split_pages,
[os.path.join(work_folder, '*.ocr.page.pdf'),
os.path.join(work_folder, '*.skip.page.pdf')],
extras=[log, context])
# Rasterize preview
task_rasterize_preview = main_pipeline.transform(
task_func=rasterize_preview,
input=task_split_pages,
input=task_ocr_or_skip,
filter=suffix('.page.pdf'),
output='.preview.jpg',
output_dir=work_folder,
@@ -1054,7 +1094,7 @@ def build_pipeline(options, work_folder, log, context):
# Orient
task_orient_page = main_pipeline.collate(
task_func=orient_page,
input=[task_split_pages, task_rasterize_preview],
input=[task_ocr_or_skip, task_rasterize_preview],
filter=regex(r".*/(\d{6})(\.ocr|\.skip)(?:\.page\.pdf|\.preview\.jpg)"),
output=os.path.join(work_folder, r'\1\2.oriented.pdf'),
extras=[log, context])