From 599d8897039dd29419a293d3a1289a58c70b5b0e Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 14:39:12 -0800 Subject: [PATCH 1/7] Implement "perfect reconstruction" - transfer page and watermark OCR layer Works, does not account for changes to clean/deskew, etc. Surprisingly, it works. PyPDF2 fixes since last attempt? --- ocrmypdf/main.py | 35 ++++++++++++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index 99f64602..2ba4da9b 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -569,7 +569,7 @@ def select_image_for_pdf( @collate( input=[select_image_for_pdf, ocr_tesseract_hocr], filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"), - output=os.path.join(work_folder, r'\1.rendered.pdf'), + output=os.path.join(work_folder, r'\1.hocr.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) def render_hocr_page( infiles, @@ -584,7 +584,7 @@ def render_hocr_page( dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample)) hocrtransform = HocrTransform(hocr, dpi) - hocrtransform.to_pdf(output_file, imageFileName=image, + hocrtransform.to_pdf(output_file, imageFileName=None, showBoundingboxes=False, invisibleText=True) @@ -612,6 +612,35 @@ def render_hocr_debug_page( showBoundingboxes=True, invisibleText=False) +@active_if(options.pdf_renderer == 'hocr') +@collate( + input=[render_hocr_page, split_pages], + filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.ocr\.page\.pdf)"), + output=os.path.join(work_folder, r'\1.rendered.pdf'), + extras=[_log, _pdfinfo, _pdfinfo_lock]) +def add_text_layer( + infiles, + output_file, + log, + pdfinfo, + pdfinfo_lock): + text = next(ii for ii in infiles if ii.endswith('.hocr.pdf')) + image = next(ii for ii in infiles if ii.endswith('.ocr.page.pdf')) + + pdf_output = pypdf.PdfFileWriter() + + pdf_text = pypdf.PdfFileReader(open(text, "rb")) + pdf_image = pypdf.PdfFileReader(open(image, "rb")) + + page = pdf_text.getPage(0) + page.mergePage(pdf_image.getPage(0)) + + pdf_output.addPage(page) + + with open(output_file, "wb") as out: + pdf_output.write(out) + + @active_if(options.pdf_renderer == 'tesseract') @collate( input=[preprocess_clean, split_pages], @@ -703,7 +732,7 @@ def skip_page( @merge( - input=[render_hocr_page, render_hocr_debug_page, skip_page, + input=[add_text_layer, render_hocr_debug_page, skip_page, tesseract_ocr_and_render_pdf, generate_postscript_stub], output=os.path.join(work_folder, 'merged.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) From 7067110308bbcbca4cbb26249cb9313b2d52fb37 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 14:54:45 -0800 Subject: [PATCH 2/7] Add safety check to prevent merge from running when not sensible --- ocrmypdf/main.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index 2ba4da9b..bba554bf 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -247,6 +247,10 @@ if options.clean and not options.clean_final \ "Tesseract PDF renderer cannot render --clean pages without " "also performing --clean-final, so --clean-final is assumed.") +lossless_reconstruction = False +if options.pdf_renderer == 'hocr': + if not options.deskew and not options.clean_final: + lossless_reconstruction = True # ---------- # Logging @@ -613,6 +617,7 @@ def render_hocr_debug_page( @active_if(options.pdf_renderer == 'hocr') +@active_if(lossless_reconstruction) @collate( input=[render_hocr_page, split_pages], filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.ocr\.page\.pdf)"), From f3e04cce56cd153d3f888d25269e4b7471619910 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 14:56:16 -0800 Subject: [PATCH 3/7] Update pipeline.svg --- pipeline.svg | 320 +++++++++++++++++++++++++++------------------------ 1 file changed, 169 insertions(+), 151 deletions(-) diff --git a/pipeline.svg b/pipeline.svg index 1a2e2f44..73b5cb8e 100644 --- a/pipeline.svg +++ b/pipeline.svg @@ -4,232 +4,250 @@ - - + + Pipeline: - + clustertasks - -Pipeline: + +Pipeline: t0 - - - - -repair_pdf + + + + +ocrmypdf.main.repair_pdf t1 - - -split_pages + + +ocrmypdf.main.split_pages t0->t1 - - + + - -t10 - - - - -generate_postscript_stub + +t11 + + + + +ocrmypdf.main.generate_postscript_stub - -t0->t10 - - + +t0->t11 + + t2 - - - - -rasterize_with_ghostscript + + + + +ocrmypdf.main.rasterize_with_ghostscript t1->t2 - - - - -t11 - - - - -skip_page - - -t1->t11 - - + + -t9 - - - - -tesseract_ocr_and_render_pdf +t9 + + + + +ocrmypdf.main.add_text_layer -t1->t9 - - +t1->t9 + + + + +t12 + + + + +ocrmypdf.main.skip_page + + +t1->t12 + + + + +t10 + + + + +ocrmypdf.main.tesseract_ocr_and_render_pdf + + +t1->t10 + + t3 - - - - -preprocess_deskew + + + + +ocrmypdf.main.preprocess_deskew t2->t3 - - + + t6 - - - - -select_image_for_pdf + + + + +ocrmypdf.main.select_image_for_pdf t2->t6 - - + + t4 - - - - -preprocess_clean + + + + +ocrmypdf.main.preprocess_clean t3->t4 - - + + t3->t6 - - + + t4->t6 - - + + t5 - - - - -ocr_tesseract_hocr + + + + +ocrmypdf.main.ocr_tesseract_hocr t4->t5 - - + + - -t4->t9 - - + +t4->t10 + + t7 - - - - -render_hocr_page + + + + +ocrmypdf.main.render_hocr_page t6->t7 - - + + -t8 - - - - -render_hocr_debug_page +t8 + + + + +ocrmypdf.main.render_hocr_debug_page -t6->t8 - - +t6->t8 + + t5->t7 - - + + -t5->t8 - - +t5->t8 + + - -t12 - - -merge_pages - - -t7->t12 - - - - -t8->t12 - - - - -t11->t12 - - - - -t9->t12 - - - - -t10->t12 - - + +t7->t9 + + t13 - - - - -validate_pdfa + + +ocrmypdf.main.merge_pages + + +t9->t13 + + + + +t8->t13 + + -t12->t13 - - +t12->t13 + + + + +t10->t13 + + + + +t11->t13 + + + + +t14 + + + + +ocrmypdf.main.copy_final + + +t13->t14 + + From dc0fb25e64d8387e454988ba492a1693a8ea6a12 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 15:16:47 -0800 Subject: [PATCH 4/7] Render hocr page: no longer needs an image as input --- ocrmypdf/main.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index bba554bf..7e49834d 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -249,7 +249,7 @@ if options.clean and not options.clean_final \ lossless_reconstruction = False if options.pdf_renderer == 'hocr': - if not options.deskew and not options.clean_final: + if not options.deskew and not options.clean_final and not options.force_ocr: lossless_reconstruction = True # ---------- @@ -570,21 +570,19 @@ def select_image_for_pdf( @active_if(options.pdf_renderer == 'hocr') -@collate( - input=[select_image_for_pdf, ocr_tesseract_hocr], - filter=regex(r".*/(\d{6})(?:\.image|\.hocr)"), - output=os.path.join(work_folder, r'\1.hocr.pdf'), +@transform( + input=ocr_tesseract_hocr, + filter=suffix('.hocr'), + output='.hocr.pdf', extras=[_log, _pdfinfo, _pdfinfo_lock]) def render_hocr_page( - infiles, + input_file, output_file, log, pdfinfo, pdfinfo_lock): - hocr = next(ii for ii in infiles if ii.endswith('.hocr')) - image = next(ii for ii in infiles if ii.endswith('.image')) - - pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock) + hocr = input_file + pageinfo = get_pageinfo(hocr, pdfinfo, pdfinfo_lock) dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample)) hocrtransform = HocrTransform(hocr, dpi) From 62728205b6ff770f681ce9af19110adac870fa5d Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 15:38:08 -0800 Subject: [PATCH 5/7] Implement image+text merging in other cases 5 failed, 28 passed failures: test_oversample[hocr], test_skip_ocr, test_skip_big, test_maximum_options[hocr], test_blank_input_pdf, --- ocrmypdf/main.py | 34 ++++- pipeline.svg | 331 ++++++++++++++++++++++-------------------- requirements.txt | 3 +- test_requirements.txt | 1 - 4 files changed, 204 insertions(+), 165 deletions(-) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index 7e49834d..4335931e 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -11,6 +11,7 @@ import warnings import multiprocessing import atexit import textwrap +import img2pdf import PyPDF2 as pypdf from PIL import Image @@ -569,6 +570,32 @@ def select_image_for_pdf( re_symlink(image, output_file) +@active_if(options.pdf_renderer == 'hocr') +@collate( + input=[select_image_for_pdf, split_pages], + filter=regex(r".*/(\d{6})(?:\.image|\.page\.pdf)"), + output=os.path.join(work_folder, r'\1.image-layer.pdf'), + extras=[_log, _pdfinfo, _pdfinfo_lock]) +def select_image_layer( + infiles, + output_file, + log, + pdfinfo, + pdfinfo_lock): + + page_pdf = next(ii for ii in infiles if ii.endswith('.page.pdf')) + image = next(ii for ii in infiles if ii.endswith('.image')) + + if lossless_reconstruction: + re_symlink(page_pdf, output_file) + else: + pageinfo = get_pageinfo(image, pdfinfo, pdfinfo_lock) + dpi = round(max(pageinfo['xres'], pageinfo['yres'], options.oversample)) + pdf_bytes = img2pdf.convert([image], dpi=dpi) + with open(output_file, 'wb') as pdf: + pdf.write(pdf_bytes) + + @active_if(options.pdf_renderer == 'hocr') @transform( input=ocr_tesseract_hocr, @@ -615,10 +642,9 @@ def render_hocr_debug_page( @active_if(options.pdf_renderer == 'hocr') -@active_if(lossless_reconstruction) @collate( - input=[render_hocr_page, split_pages], - filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.ocr\.page\.pdf)"), + input=[render_hocr_page, select_image_layer], + filter=regex(r".*/(\d{6})(?:\.hocr\.pdf|\.image-layer\.pdf)"), output=os.path.join(work_folder, r'\1.rendered.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) def add_text_layer( @@ -628,7 +654,7 @@ def add_text_layer( pdfinfo, pdfinfo_lock): text = next(ii for ii in infiles if ii.endswith('.hocr.pdf')) - image = next(ii for ii in infiles if ii.endswith('.ocr.page.pdf')) + image = next(ii for ii in infiles if ii.endswith('.image-layer.pdf')) pdf_output = pypdf.PdfFileWriter() diff --git a/pipeline.svg b/pipeline.svg index 73b5cb8e..3c639f86 100644 --- a/pipeline.svg +++ b/pipeline.svg @@ -19,235 +19,248 @@ -ocrmypdf.main.repair_pdf +repair_pdf t1 -ocrmypdf.main.split_pages +split_pages t0->t1 - -t11 + +t12 -ocrmypdf.main.generate_postscript_stub +generate_postscript_stub - -t0->t11 + +t0->t12 t2 - - - - -ocrmypdf.main.rasterize_with_ghostscript + + + + +rasterize_with_ghostscript t1->t2 - - + + - -t9 - - - - -ocrmypdf.main.add_text_layer + +t7 + + + + +select_image_layer - -t1->t9 - - + +t1->t7 + + - -t12 - - - - -ocrmypdf.main.skip_page + +t13 + + + + +skip_page - -t1->t12 - - + +t1->t13 + + - -t10 - - - - -ocrmypdf.main.tesseract_ocr_and_render_pdf + +t11 + + + + +tesseract_ocr_and_render_pdf - -t1->t10 - - + +t1->t11 + + t3 - - - - -ocrmypdf.main.preprocess_deskew + + + + +preprocess_deskew t2->t3 - - + + -t6 - - - - -ocrmypdf.main.select_image_for_pdf +t6 + + + + +select_image_for_pdf -t2->t6 - - +t2->t6 + + t4 - - - - -ocrmypdf.main.preprocess_clean + + + + +preprocess_clean t3->t4 - - + + -t3->t6 - - - - -t4->t6 - - +t3->t6 + + -t5 - - - - -ocrmypdf.main.ocr_tesseract_hocr +t5 + + + + +ocr_tesseract_hocr -t4->t5 - - +t4->t5 + + - -t4->t10 - - + +t4->t6 + + - -t7 - - - - -ocrmypdf.main.render_hocr_page - - -t6->t7 - - + +t4->t11 + + -t8 - - - - -ocrmypdf.main.render_hocr_debug_page - - -t6->t8 - - - - -t5->t7 - - +t8 + + + + +render_hocr_page -t5->t8 - - +t5->t8 + + - -t7->t9 - - + +t9 + + + + +render_hocr_debug_page - -t13 - - -ocrmypdf.main.merge_pages + +t5->t9 + + - -t9->t13 - - + +t10 + + + + +add_text_layer - -t8->t13 - - + +t8->t10 + + - -t12->t13 - - + +t6->t7 + + - -t10->t13 - - + +t6->t9 + + - -t11->t13 - - + +t7->t10 + + t14 - - - - -ocrmypdf.main.copy_final + + +merge_pages + + +t10->t14 + + + + +t9->t14 + + -t13->t14 - - +t13->t14 + + + + +t11->t14 + + + + +t12->t14 + + + + +t15 + + + + +copy_final + + +t14->t15 + + diff --git a/requirements.txt b/requirements.txt index 7cf67a5b..5fc71a98 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ ruffus>=2.6.3 Pillow>=2.4.0 reportlab>=3.1.44 -PyPDF2>=1.25.1 \ No newline at end of file +PyPDF2>=1.25.1 +img2pdf>=0.1.5 \ No newline at end of file diff --git a/test_requirements.txt b/test_requirements.txt index 0cdf49af..7c419275 100644 --- a/test_requirements.txt +++ b/test_requirements.txt @@ -1,2 +1 @@ -img2pdf>=0.1.5 pytest>=2.7.2 \ No newline at end of file From fc0479f1100a3baea8726b477546d5bdf9c798c3 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 15:44:39 -0800 Subject: [PATCH 6/7] Fix all but test_oversample[hocr] --- ocrmypdf/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ocrmypdf/main.py b/ocrmypdf/main.py index 4335931e..50c2a0fb 100755 --- a/ocrmypdf/main.py +++ b/ocrmypdf/main.py @@ -573,7 +573,7 @@ def select_image_for_pdf( @active_if(options.pdf_renderer == 'hocr') @collate( input=[select_image_for_pdf, split_pages], - filter=regex(r".*/(\d{6})(?:\.image|\.page\.pdf)"), + filter=regex(r".*/(\d{6})(?:\.image|\.ocr\.page\.pdf)"), output=os.path.join(work_folder, r'\1.image-layer.pdf'), extras=[_log, _pdfinfo, _pdfinfo_lock]) def select_image_layer( From 360acd1e2cdc01156c08811f744d7280ea1ae785 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Fri, 15 Jan 2016 15:55:23 -0800 Subject: [PATCH 7/7] Adjust test_oversample test case Add -f to force generation of the background image at the desired oversample resolution. Our new behavior is to only send the oversampled image to Tesseract while leaving the main page intact unless asked to deskew, clean, etc. --- tests/test_main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test_main.py b/tests/test_main.py index 23056c24..1e7edb74 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -199,6 +199,7 @@ def test_override_metadata(spoof_tesseract_noop): def test_oversample(spoof_tesseract_cache, renderer): oversampled_pdf = check_ocrmypdf( 'skew.pdf', 'test_oversample_%s.pdf' % renderer, '--oversample', '300', + '-f', '--pdf-renderer', renderer, env=spoof_tesseract_cache) pdfinfo = pdf_get_all_pageinfo(oversampled_pdf)