diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index 99f64602..50c2a0fb 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -11,6 +11,7 @@ import warnings import multiprocessing import atexit import textwrap +import img2pdf import PyPDF2 as pypdf from PIL import Image @@ -247,6 +248,10 @@ if options.clean and not options.clean_final \ "Tesseract PDF renderer cannot render --clean pages without " "also performing --clean-final, so --clean-final is assumed.") +lossless_reconstruction = False +if options.pdf_renderer == 'hocr': + if not options.deskew and not options.clean_final and not options.force_ocr: + lossless_reconstruction = True # ---------- # Logging @@ -567,24 +572,48 @@ def select_image_for_pdf( @active_if(options.pdf_renderer == 'hocr') @collate( - input=[select_image_for_pdf, ocr_tesseract_hocr], - filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"), - output=os.path.join(work_folder, r'\1.rendered.pdf'), + input=[select_image_for_pdf, split_pages], + filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.page\.pdf)"), + output=os.path.join(work_folder, r'\1.image-layer.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) -def render_hocr_page( +def select_image_layer( infiles, output_file, log, pdfinfo, pdfinfo_lock): - hocr = next(ii for ii in infiles if ii.endswith('.hocr')) + + page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf')) image = next(ii for ii in infiles if ii.endswith('.image')) - pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock) + if lossless_reconstruction: + re_symlink(page_pdf, output_file) + else: + pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock) + dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample)) + pdf_bytes = img2pdf.convert([image], dpi=dpi) + with open(output_file, 'wb') as pdf: + pdf.write(pdf_bytes) + + +@active_if(options.pdf_renderer == 'hocr') +@transform( + input=ocr_tesseract_hocr, + filter=suffix('.hocr'), + output='.hocr.pdf', + extras=[_log, _pdfinfo, _pdfinfo_lock]) +def render_hocr_page( + input_file, + output_file, + log, + pdfinfo, + pdfinfo_lock): + hocr = input_file + pageinfo = get_pageinfo(hocr, pdfinfo, pdfinfo_lock) dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample)) hocrtransform = HocrTransform(hocr, dpi) - hocrtransform.to_pdf(output_file, imageFileName=image, + hocrtransform.to_pdf(output_file, imageFileName=None, showBoundingboxes=False, invisibleText=True) @@ -612,6 +641,35 @@ def render_hocr_debug_page( showBoundingboxes=True, invisibleText=False) +@active_if(options.pdf_renderer == 'hocr') +@collate( + input=[render_hocr_page, select_image_layer], + filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.image-layer\.pdf)"), + output=os.path.join(work_folder, r'\1.rendered.pdf'), + extras=[_log, _pdfinfo, _pdfinfo_lock]) +def add_text_layer( + infiles, + output_file, + log, + pdfinfo, + pdfinfo_lock): + text = next(ii for ii in infiles if ii.endswith('.hocr.pdf')) + image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf')) + + pdf_output = pypdf.PdfFileWriter() + + pdf_text = pypdf.PdfFileReader(open(text, "rb")) + pdf_image = pypdf.PdfFileReader(open(image, "rb")) + + page = pdf_text.getPage(0) + page.mergePage(pdf_image.getPage(0)) + + pdf_output.addPage(page) + + with open(output_file, "wb") as out: + pdf_output.write(out) + + @active_if(options.pdf_renderer == 'tesseract') @collate( input=[preprocess_clean, split_pages], @@ -703,7 +761,7 @@ def skip_page( @merge( - input=[render_hocr_page, render_hocr_debug_page, skip_page, + input=[add_text_layer, render_hocr_debug_page, skip_page, tesseract_ocr_and_render_pdf, generate_postscript_stub], output=os.path.join(work_folder, 'merged.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) diff --git a/pipeline.svg b/pipeline.svg index 1a2e2f44..3c639f86 100644 --- a/pipeline.svg +++ b/pipeline.svg @@ -4,232 +4,263 @@ - - + + Pipeline: - + clustertasks - -Pipeline: + +Pipeline: t0 - - - - -repair_pdf + + + + +repair_pdf t1 - - -split_pages + + +split_pages t0->t1 - - + + - -t10 - - - - -generate_postscript_stub + +t12 + + + + +generate_postscript_stub - -t0->t10 - - + +t0->t12 + + t2 - - - - -rasterize_with_ghostscript + + + + +rasterize_with_ghostscript t1->t2 - - + + + + +t7 + + + + +select_image_layer + + +t1->t7 + + + + +t13 + + + + +skip_page + + +t1->t13 + + -t11 - - - - -skip_page +t11 + + + + +tesseract_ocr_and_render_pdf -t1->t11 - - - - -t9 - - - - -tesseract_ocr_and_render_pdf - - -t1->t9 - - +t1->t11 + + t3 - - - - -preprocess_deskew + + + + +preprocess_deskew t2->t3 - - + + -t6 - - - - -select_image_for_pdf +t6 + + + + +select_image_for_pdf -t2->t6 - - +t2->t6 + + t4 - - - - -preprocess_clean + + + + +preprocess_clean t3->t4 - - + + -t3->t6 - - - - -t4->t6 - - +t3->t6 + + -t5 - - - - -ocr_tesseract_hocr +t5 + + + + +ocr_tesseract_hocr -t4->t5 - - +t4->t5 + + - -t4->t9 - - + +t4->t6 + + - -t7 - - - - -render_hocr_page - - -t6->t7 - - + +t4->t11 + + -t8 - - - - -render_hocr_debug_page - - -t6->t8 - - - - -t5->t7 - - +t8 + + + + +render_hocr_page -t5->t8 - - +t5->t8 + + - -t12 - - -merge_pages + +t9 + + + + +render_hocr_debug_page - -t7->t12 - - + +t5->t9 + + - -t8->t12 - - + +t10 + + + + +add_text_layer - -t11->t12 - - + +t8->t10 + + - -t9->t12 - - + +t6->t7 + + - -t10->t12 - - + +t6->t9 + + - -t13 - - - - -validate_pdfa + +t7->t10 + + - -t12->t13 - - + +t14 + + +merge_pages + + +t10->t14 + + + + +t9->t14 + + + + +t13->t14 + + + + +t11->t14 + + + + +t12->t14 + + + + +t15 + + + + +copy_final + + +t14->t15 + + diff --git a/requirements.txt b/requirements.txt index 7cf67a5b..5fc71a98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ ruffus>=2.6.3 Pillow>=2.4.0 reportlab>=3.1.44 -PyPDF2>=1.25.1 \ No newline at end of file +PyPDF2>=1.25.1 +img2pdf>=0.1.5 \ No newline at end of file diff --git a/test_requirements.txt b/test_requirements.txt index 0cdf49af..7c419275 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1 @@ -img2pdf>=0.1.5 pytest>=2.7.2 \ No newline at end of file diff --git a/tests/test_main.py b/tests/test_main.py index 23056c24..1e7edb74 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -199,6 +199,7 @@ def test_override_metadata(spoof_tesseract_noop): def test_oversample(spoof_tesseract_cache, renderer): oversampled_pdf = check_ocrmypdf( 'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300', + '-f', '--pdf-renderer', renderer, env=spoof_tesseract_cache) pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)