From 88ff091ccecfb6c185426ca0d4c874a7ab00b7d8 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 1 May 2018 13:24:20 -0700 Subject: [PATCH] Unify tesseract and sandwich renderer paths Since the new weaving method copies the font and content stream from the Tesseract PDF, it doesn't matter if Tesseract happens to have an image or not. If Tesseract is text-only capable we use that feature for efficiency, but ignore the image either way. --- docs/pipeline.svg | 823 ++++++++++++++++----------------- src/ocrmypdf/exec/tesseract.py | 2 +- src/ocrmypdf/pipeline.py | 56 +-- 3 files changed, 407 insertions(+), 474 deletions(-) diff --git a/docs/pipeline.svg b/docs/pipeline.svg index bb461c94..8361bba4 100644 --- a/docs/pipeline.svg +++ b/docs/pipeline.svg @@ -4,470 +4,453 @@ - - + + Pipeline: - + clustertasks - -Pipeline: + +Pipeline: t0 - - - - -ocrmypdf.pipeline.triage + + + + +ocrmypdf.pipeline.triage t1 - - - - -ocrmypdf.pipeline.repair_and_parse_pdf + + + + +ocrmypdf.pipeline.repair_and_parse_pdf t0->t1 - - + + t2 - - -ocrmypdf.pipeline.pre_split_pages + + +ocrmypdf.pipeline.marker_pages t1->t2 - - - - - -t20 - - - - -ocrmypdf.pipeline.generate_postscript_stub - - - -t1->t20 - - - - - -t22 - - -ocrmypdf.pipeline.merge_pages - - - -t1->t22 - - - - - -t3 - - - - -ocrmypdf.pipeline.split_page - - - -t2->t3 - - - - - -t4 - - -ocrmypdf.pipeline.ocr_or_skip - - - -t3->t4 - - - - - -t5 - - - - -ocrmypdf.pipeline.rasterize_preview - - - -t4->t5 - - - - - -t6 - - - - -ocrmypdf.pipeline.orient_page - - - -t4->t6 - - - - - -t5->t6 - - - - - -t7 - - - - -ocrmypdf.pipeline.rasterize_with_ghostscript - - - -t6->t7 - - + + - -t17 - - - - -ocrmypdf.pipeline.ocr_tesseract_textonly_pdf - - - -t6->t17 - - - - -t14 - -ocrmypdf.pipeline.select_image_layer +t17 + + + + +ocrmypdf.pipeline.weave_layers - - -t6->t14 - - - - - -t21 - - - - -ocrmypdf.pipeline.skip_page - - - -t6->t21 - - - - - -t19 - - - - -ocrmypdf.pipeline.ocr_tesseract_and_render_pdf - - - -t6->t19 - - - - - -t8 - - - - -ocrmypdf.pipeline.preprocess_remove_background - - - -t7->t8 - - - - - -t13 - -ocrmypdf.pipeline.select_visible_page_image - - - -t7->t13 - - - - - -t9 - - - - -ocrmypdf.pipeline.preprocess_deskew - - - -t8->t9 - - - - - -t8->t13 - - - - - -t10 - - - - -ocrmypdf.pipeline.preprocess_clean - - - -t9->t10 - - - - - -t9->t13 - - - - - -t11 - - - - -ocrmypdf.pipeline.select_ocr_image - - - -t10->t11 - - - - - -t10->t13 - - - - - -t12 - - - - -ocrmypdf.pipeline.ocr_tesseract_hocr - - - -t11->t12 - - - - - -t11->t17 - - - - - -t15 - - - - -ocrmypdf.pipeline.render_hocr_page - - - -t12->t15 - - - - - -t16 - - - - -ocrmypdf.pipeline.render_hocr_debug_page - - - -t12->t16 - - - - - -t23 - - -ocrmypdf.pipeline.merge_sidecars - - - -t12->t23 - - + + +t1->t17 + + t18 - - - - -ocrmypdf.pipeline.combine_layers + + + + +ocrmypdf.pipeline.generate_postscript_stub - - -t15->t18 - - - - - -t17->t18 - - - - - -t17->t23 - - - - - -t13->t14 - - - - + -t13->t16 - - +t1->t18 + + - + + +t20 + + +ocrmypdf.pipeline.metadata_fixup + + -t13->t19 - - +t1->t20 + + - - -t14->t18 - - + + +t21 + + +ocrmypdf.pipeline.merge_pages - - -t18->t22 - - + + +t1->t21 + + + + + +t3 + + +ocrmypdf.pipeline.ocr_or_skip + + + +t2->t3 + + + + + +t4 + + + + +ocrmypdf.pipeline.rasterize_preview + + + +t3->t4 + + + + + +t5 + + + + +ocrmypdf.pipeline.orient_page + + + +t3->t5 + + + + + +t4->t5 + + + + + +t6 + + + + +ocrmypdf.pipeline.rasterize_with_ghostscript + + + +t5->t6 + + + + + +t16 + + + + +ocrmypdf.pipeline.ocr_tesseract_textonly_pdf + + + +t5->t16 + + + + + +t13 + +ocrmypdf.pipeline.select_image_layer + + + +t5->t13 + + + + + +t19 + + + + +ocrmypdf.pipeline.skip_page + + + +t5->t19 + + + + + +t7 + + + + +ocrmypdf.pipeline.preprocess_remove_background + + + +t6->t7 + + + + + +t12 + +ocrmypdf.pipeline.select_visible_page_image + + + +t6->t12 + + + + + +t8 + + + + +ocrmypdf.pipeline.preprocess_deskew + + + +t7->t8 + + + + + +t7->t12 + + + + + +t9 + + + + +ocrmypdf.pipeline.preprocess_clean + + + +t8->t9 + + + + + +t8->t12 + + + + + +t10 + + + + +ocrmypdf.pipeline.select_ocr_image + + + +t9->t10 + + + + + +t9->t12 + + + + + +t11 + + + + +ocrmypdf.pipeline.ocr_tesseract_hocr + + + +t10->t11 + + + + + +t10->t16 + + + + + +t14 + + + + +ocrmypdf.pipeline.render_hocr_page + + + +t11->t14 + + + + + +t15 + + + + +ocrmypdf.pipeline.render_hocr_debug_page + + + +t11->t15 + + + + + +t22 + + +ocrmypdf.pipeline.merge_sidecars + + + +t11->t22 + + + + + +t14->t17 + + + + + +t16->t17 + + - -t16->t22 - - - - - -t21->t22 - - - - - -t19->t22 - - - - -t19->t23 - - +t16->t22 + + - - -t20->t22 - - + + +t12->t13 + + - - -t24 - - -ocrmypdf.pipeline.copy_final + + +t12->t15 + + - + + +t13->t17 + + + + + +t17->t20 + + + + + +t18->t20 + + + + + +t18->t21 + + + + + +t23 + + +ocrmypdf.pipeline.copy_final + + -t22->t24 - - +t20->t23 + + + + + +t15->t21 + + + + + +t19->t21 + + + + + +t21->t23 + + diff --git a/src/ocrmypdf/exec/tesseract.py b/src/ocrmypdf/exec/tesseract.py index b3f96093..64dfb2ae 100644 --- a/src/ocrmypdf/exec/tesseract.py +++ b/src/ocrmypdf/exec/tesseract.py @@ -316,7 +316,7 @@ def generate_pdf(*, input_image, skip_pdf, output_pdf, output_text, if pagesegmode is not None: args_tesseract.extend([psm(), str(pagesegmode)]) - if text_only: + if text_only and has_textonly_pdf(): args_tesseract.extend(['-c', 'textonly_pdf=1']) if user_words: diff --git a/src/ocrmypdf/pipeline.py b/src/ocrmypdf/pipeline.py index a90e3a53..61846147 100644 --- a/src/ocrmypdf/pipeline.py +++ b/src/ocrmypdf/pipeline.py @@ -939,40 +939,6 @@ def weave_layers( # pdf_output.write(out) -def ocr_tesseract_and_render_pdf( - infiles, - outfiles, - log, - context): - options = context.get_options() - input_image = next((ii for ii in infiles if ii.endswith('.image')), '') - input_pdf = next((ii for ii in infiles if ii.endswith('.pdf'))) - output_pdf = next((ii for ii in outfiles if ii.endswith('.pdf'))) - output_text = next((ii for ii in outfiles if ii.endswith('.txt'))) - - if not input_image: - # Skipping this page - re_symlink(input_pdf, output_pdf, log) - with open(output_text, 'w') as f: - f.write('[skipped page]') - return - - tesseract.generate_pdf( - input_image=input_image, - skip_pdf=input_pdf, - output_pdf=output_pdf, - output_text=output_text, - language=options.language, - engine_mode=options.tesseract_oem, - text_only=False, - tessconfig=options.tesseract_config, - timeout=options.tesseract_timeout, - pagesegmode=options.tesseract_pagesegmode, - user_words=options.user_words, - user_patterns=options.user_patterns, - log=log) - - def ocr_tesseract_textonly_pdf( infiles, outfiles, @@ -1025,9 +991,7 @@ def get_pdfmark(base_pdf, options): if options.subject: pdfmark['/Subject'] = options.subject - if options.pdf_renderer == 'tesseract': - renderer_tag = 'OCR+PDF' - elif options.pdf_renderer == 'sandwich': + if options.pdf_renderer == 'sandwich': renderer_tag = 'OCR-PDF' else: renderer_tag = 'OCR' @@ -1438,7 +1402,8 @@ def build_pipeline(options, work_folder, log, context): os.path.join(work_folder, r'\1.text.txt')], extras=[log, context]) task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"') - task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich') + task_ocr_tesseract_textonly_pdf.active_if( + options.pdf_renderer == 'tesseract' or options.pdf_renderer == 'sandwich') task_weave_layers = main_pipeline.collate( task_func=weave_layers, @@ -1451,19 +1416,6 @@ def build_pipeline(options, work_folder, log, context): output=os.path.join(work_folder, r'layers.rendered.pdf'), extras=[log, context]) task_weave_layers.graphviz(fillcolor='"#00cc66"') - task_weave_layers.active_if(options.pdf_renderer == 'hocr' or - options.pdf_renderer == 'sandwich') - - # Tesseract OCR+PDF - task_ocr_tesseract_and_render_pdf = main_pipeline.collate( - task_func=ocr_tesseract_and_render_pdf, - input=[task_select_visible_page_image, task_orient_page], - filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.oriented\.pdf)"), - output=[os.path.join(work_folder, r'\1.rendered.pdf'), - os.path.join(work_folder, r'\1.rendered.txt')], - extras=[log, context]) - task_ocr_tesseract_and_render_pdf.graphviz(fillcolor='"#66ccff"') - task_ocr_tesseract_and_render_pdf.active_if(options.pdf_renderer == 'tesseract') # PDF/A task_generate_postscript_stub = main_pipeline.transform( @@ -1498,7 +1450,6 @@ def build_pipeline(options, work_folder, log, context): input=[task_repair_and_parse_pdf, task_render_hocr_debug_page, task_skip_page, - task_ocr_tesseract_and_render_pdf, task_generate_postscript_stub], output=os.path.join(work_folder, 'merged.pdf'), extras=[log, context]) @@ -1507,7 +1458,6 @@ def build_pipeline(options, work_folder, log, context): task_merge_sidecars = main_pipeline.merge( task_func=merge_sidecars, input=[task_ocr_tesseract_hocr, - task_ocr_tesseract_and_render_pdf, task_ocr_tesseract_textonly_pdf], output=options.sidecar, extras=[log, context])