diff --git a/docs/advanced.rst b/docs/advanced.rst index 7b1c9c95..85e6be0d 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -42,10 +42,9 @@ For example, if you are testing tesseract 4.00 and don't wish to disturb your te env \ OCRMYPDF_TESSERACT=/home/user/src/tesseract4/api/tesseract \ TESSDATA_PREFIX=/home/user/src/tesseract4 \ - ocrmypdf --pdf-renderer tess4 --tesseract-oem 2 input.pdf output.pdf + ocrmypdf --tesseract-oem 2 input.pdf output.pdf * ``TESSDATA_PREFIX`` directs tesseract 4.0 to use LSTM training data. This is a tesseract environment variable. -* ``--pdf-renderer tess4`` takes advantage of new tesseract 4.0 PDF renderer in OCRmyPDF. (Tesseract 4.0 only.) * ``--tesseract-oem 1`` requests tesseract 4.0's new LSTM engine. (Tesseract 4.0 only.) Overriding other support programs @@ -95,33 +94,29 @@ rendering Creating a new PDF from other data (such as an existing PDF). -OCRmyPDF has three PDF renderers: ``hocr``, ``tesseract`` and ``tess4``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` always selects ``hocr``. +OCRmyPDF has three PDF renderers: ``sandwich``, ``hocr``, ``tesseract``. The renderer may be selected using ``--pdf-renderer``. The default is ``auto`` which lets OCRmyPDF select the renderer to use. Currently, ``auto`` selects ``sandwich`` for Tesseract 3.05.01, and newer, ``hocr`` for older versions of Tesseract. + +The ``sandwich`` renderer +""""""""""""""""""""""""" + +The ``sandwich`` renderer uses Tesseract's new text-only PDF feature, which produces a PDF page that lays out the OCR in invisible text. This page is then "sandwiched" onto the original PDF page, allowing lossless application of OCR even to PDF pages that contain other vector objects. + +When image preprocessing features like ``--deskew`` are used, the original PDF will be rendered as a full page and the OCR layer will be placed on top. + +This renderer requires Tesseract 3.05.01 or newer. The ``hocr`` renderer """"""""""""""""""""" -The ``hocr`` renderer is the default because it works in most cases. In this mode the whole PDF is rasterized, the raster image is run through OCR to generate a .hocr file, which is an HTML-like file that specifies the location of all identified words. - -The .hocr file is then rendered as a PDF and merged with the image layer. - -The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF. - -This is the only option for tesseract 3.02 and older. +The ``hocr`` renderer works with older versions of Tesseract. The image layer is copied from the original PDF page if possible, avoiding potentially lossy transcoding or loss of other PDF information. If preprocessing is specified, then the image layer is a new PDF. +This works in all versions of Tesseract. The ``tesseract`` renderer """""""""""""""""""""""""" -The tesseract renderer uses tesseract's capability to produce a PDF directly. In version 3, tesseract automatically combined the image layer and text, meaning that this mode *always* transcodes and loses potentially loses image quality and other PDF information. +The ``tesseract`` renderer creates a PDF with the image and text layers precomposed, meaning that it always transcodes, loses image quality and rasterizes and vector objects. It does a better job on non-Latin text and document structure than ``hocr``. -It does a much better job on non-Latin text. +If a PDF created with this renderer using Tesseract versions older than 3.05.00 is then passed through Ghostscript's pdfwrite feature, the OCR text *may* be corrupted. The ``--output-type=pdfa`` argument will produce a warning in this situation. -In a future release this will become the "tess3" renderer and ultimately will be dropped. - - -The ``tess4`` renderer -"""""""""""""""""""""" - -The tess4 renderer uses tesseract 4.00 alpha's text-only PDF feature added in January 2017. This combines the advantages of the tesseract and hocr renderers, transcoding the image layer only if required by preprocessing options. - -Ghostscript PDF/A still sometimes inserts spaces between words when the tess4 renderer is used, affecting search quality. ``--output-pdf pdf`` may be used to avoid this issue. \ No newline at end of file +*This renderer is deprecated and will be removed whenever support for older versions of Tesseract is dropped.* diff --git a/docs/release_notes.rst b/docs/release_notes.rst index 48c11e42..f238b84a 100644 --- a/docs/release_notes.rst +++ b/docs/release_notes.rst @@ -6,6 +6,14 @@ OCRmyPDF uses `semantic versioning `_ for its command line i The OCRmyPDF package itself does not contain a public API, although it is fairly stable and breaking changes are usually timed with a major release. A future release will clearly define the stable public API. +v5.2 +---- + +- When using Tesseract 3.05.01 or newer, OCRmyPDF will select the "sandwich" PDF renderer by default, unless another PDF renderer is specified with the ``--pdf-renderer`` argument. The previous behavior was to select ``--pdf-renderer=hocr``. +- The "tesseract" PDF renderer is now deprecated, since it can cause problems with Ghostscript on Tesseract 3.05.00 +- The "tess4" PDF renderer has been renamed to "sandwich". "tess4" is now a deprecated alias for "sandwich". + + v5.1 ---- @@ -15,7 +23,7 @@ v5.1 v5.0.1 ------ -- Fixed issue #169, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf image +- Fixed issue #169, exception due to failure to create sidecar text files on some versions of Tesseract 3.04, including the jbarlow83/ocrmypdf Docker image v5.0 diff --git a/ocrmypdf/__main__.py b/ocrmypdf/__main__.py index ba1640ff..9ad7749b 100755 --- a/ocrmypdf/__main__.py +++ b/ocrmypdf/__main__.py @@ -250,15 +250,18 @@ advanced.add_argument( "3 - default.") ) advanced.add_argument( - '--pdf-renderer', choices=['auto', 'tesseract', 'hocr', 'tess4'], default='auto', + '--pdf-renderer', + choices=['auto', 'tesseract', 'hocr', 'tess4', 'sandwich'], default='auto', help="Choose OCR PDF renderer - the default option is to let OCRmyPDF " - "choose. The 'tesseract' PDF renderer is more accurate and does a " - "better job and document structure such as recognizing columns. It " - "also does a better job on non-Latin languages. However, it does " - "not work as well when older versions of Tesseract or Ghostscript " - "are installed, and some combinations of arguments to do not work " - "with --pdf-renderer tesseract. The 'tess4' PDF renderer is similar " - "to 'tesseract', requires tesseract 4, and gives superior results.") + "choose." + "auto - let OCRmyPDF choose; " + "sandwich - default renderer for Tesseract 3.05.01 and newer; " + "hocr - default renderer for older versions of Tesseract; " + "tesseract - gives better results for non-Latin languages and " + "Tesseract older than 3.05.01 but has problems with some versions " + " of Ghostscript; deprecated" + "tess4 - deprecated alias for 'sandwich'" + ) advanced.add_argument( '--tesseract-timeout', default=180.0, type=float, metavar='SECONDS', help='Give up on OCR after the timeout, but copy the preprocessed page ' @@ -312,27 +315,31 @@ def check_options_languages(options, _log): def check_options_output(options, log): if options.pdf_renderer == 'auto': - options.pdf_renderer = 'hocr' + if tesseract.has_textonly_pdf(): + options.pdf_renderer = 'sandwich' + else: + options.pdf_renderer = 'hocr' - if options.pdf_renderer in ('tesseract', 'tess4'): - if tesseract.version() < '3.05': + if options.pdf_renderer == 'tess4': + log.warning("The 'tess4' PDF renderer has been renamed to 'sandwich'. " + "Please use --pdf-renderer=sandwich.") + options.pdf_renderer = 'sandwich' + + if options.pdf_renderer == 'tesseract': + log.warning("The 'tesseract' PDF renderer is deprecated.") + if tesseract.version() < '3.05' and options.output_type == 'pdfa': log.warning( - "The setting --pdf-renderer=tesseract is not recommend for " - " use with tesseract versions less than 3.05, because it " - " produces OCR text that is incompatible with Ghostscript and " - " some other software.") - elif tesseract.version() == '4.00.00alpha': - log.warning( - "The setting --pdf-renderer={tesseract,tess4} is not" - " recommended for builds of tesseract 4.00.00alpha older than" - " February 2017. Make sure you are using a recent build.") + "For best results use --pdf-renderer=tesseract " + "--output-type=pdf to disable PDF/A generation via " + "Ghostscript, which is known to corrupt the OCR text of " + "some PDFs produced your version of Tesseract.") if options.debug_rendering and options.pdf_renderer != 'hocr': log.info( "Ignoring --debug-rendering because it requires --pdf-renderer=hocr") lossless_reconstruction = False - if options.pdf_renderer in ('hocr', 'tess4'): + if options.pdf_renderer in ('hocr', 'sandwich'): if not any((options.deskew, options.clean_final, options.force_ocr, options.remove_background)): lossless_reconstruction = True diff --git a/ocrmypdf/pipeline.py b/ocrmypdf/pipeline.py index 6031fb67..e28e1fc2 100644 --- a/ocrmypdf/pipeline.py +++ b/ocrmypdf/pipeline.py @@ -595,7 +595,7 @@ def select_image_layer( # as it accurately describes the image. It would be possible to # resample the image at this stage back to non-square DPI to more # closely resemble the input, except that the hocr renderer does not - # understand non-square DPI. The tess4 renderer would be fine. + # understand non-square DPI. The sandwich renderer would be fine. dpi = get_page_square_dpi(pageinfo, options) layout_fun = img2pdf.get_fixed_dpi_layout_fun((dpi, dpi)) @@ -812,9 +812,16 @@ def get_pdfmark(base_pdf, options): if options.subject: pdfmark['/Subject'] = options.subject - pdfmark['/Creator'] = '{0} {1} / Tesseract OCR{2} {3}'.format( + if options.pdf_renderer == 'tesseract': + renderer_tag = 'OCR+PDF' + elif options.pdf_renderer == 'sandwich': + renderer_tag = 'OCR-PDF' + else: + renderer_tag = 'OCR' + + pdfmark['/Creator'] = '{0} {1} / Tesseract {2} {3}'.format( PROGRAM_NAME, VERSION, - '+PDF' if options.pdf_renderer == 'tesseract' else '', + renderer_tag, tesseract.version()) return pdfmark @@ -1083,7 +1090,7 @@ def build_pipeline(options, work_folder, log, context): task_select_image_layer.graphviz( fillcolor='"#00cc66"', shape='diamond') task_select_image_layer.active_if( - options.pdf_renderer == 'hocr' or options.pdf_renderer == 'tess4') + options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich') task_render_hocr_page = main_pipeline.transform( task_func=render_hocr_page, @@ -1113,7 +1120,7 @@ def build_pipeline(options, work_folder, log, context): os.path.join(work_folder, r'\1.text.txt')], extras=[log, context]) task_ocr_tesseract_textonly_pdf.graphviz(fillcolor='"#ff69b4"') - task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'tess4') + task_ocr_tesseract_textonly_pdf.active_if(options.pdf_renderer == 'sandwich') if tesseract.v4(): task_ocr_tesseract_textonly_pdf.jobs_limit(2) @@ -1126,7 +1133,7 @@ def build_pipeline(options, work_folder, log, context): output=os.path.join(work_folder, r'\1.rendered.pdf'), extras=[log, context]) task_combine_layers.graphviz(fillcolor='"#00cc66"') - task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'tess4') + task_combine_layers.active_if(options.pdf_renderer == 'hocr' or options.pdf_renderer == 'sandwich') # Tesseract OCR+PDF task_ocr_tesseract_and_render_pdf = main_pipeline.collate( diff --git a/tests/spoof/tesseract_big_image_error.py b/tests/spoof/tesseract_big_image_error.py index 5ddff56b..63cbe6e4 100755 --- a/tests/spoof/tesseract_big_image_error.py +++ b/tests/spoof/tesseract_big_image_error.py @@ -22,6 +22,9 @@ def main(): elif sys.argv[1] == '--list-langs': print('List of available languages (1):\neng', file=sys.stderr) sys.exit(0) + elif sys.argv[1] == '--print-parameters': + print('A parameter list would go here', file=sys.stderr) + sys.exit(0) elif sys.argv[-2] == 'hocr': print("Image too large: (33830, 14959)\n" "Error during processing.", file=sys.stderr) diff --git a/tests/spoof/tesseract_cache.py b/tests/spoof/tesseract_cache.py index d6c31a3e..b01fc5dc 100755 --- a/tests/spoof/tesseract_cache.py +++ b/tests/spoof/tesseract_cache.py @@ -85,6 +85,12 @@ def main(): except ValueError: m.update(b'default-lang') + try: + textonly = sys.argv[sys.argv.index('-c') + 1] + m.update(textonly.encode()) + except ValueError: + m.update(b'textonly_pdf=0') + psm_arg = '' if '--psm' in sys.argv: psm_arg = '--psm' diff --git a/tests/spoof/tesseract_crash.py b/tests/spoof/tesseract_crash.py index 81d3ffdf..42b4550c 100755 --- a/tests/spoof/tesseract_crash.py +++ b/tests/spoof/tesseract_crash.py @@ -29,6 +29,9 @@ def main(): elif sys.argv[1] == '--list-langs': print('List of available languages (1):\neng', file=sys.stderr) sys.exit(0) + elif sys.argv[1] == '--print-parameters': + print('List of parameters would go here', file=sys.stderr) + sys.exit(0) elif sys.argv[-2] == 'hocr': print("KABOOM! Tesseract failed for some reason", file=sys.stderr) sys.exit(128 + signal.SIGSEGV) diff --git a/tests/spoof/tesseract_noop.py b/tests/spoof/tesseract_noop.py index e3dc357a..8ec9062a 100755 --- a/tests/spoof/tesseract_noop.py +++ b/tests/spoof/tesseract_noop.py @@ -2,6 +2,7 @@ # © 2016 James R. Barlow: github.com/jbarlow83 import sys import img2pdf +import PyPDF2 as pypdf from PIL import Image @@ -17,7 +18,7 @@ In orientation check mode, report the orientation is upright. """ -VERSION_STRING = '''tesseract 3.04.00 +VERSION_STRING = '''tesseract 3.05.01 leptonica-1.72 libjpeg 8d : libpng 1.6.19 : libtiff 4.0.6 : zlib 1.2.5 SPOOFED @@ -53,6 +54,10 @@ def main(): elif sys.argv[1] == '--list-langs': print('List of available languages (1):\neng', file=sys.stderr) sys.exit(0) + elif sys.argv[1] == '--print-parameters': + print("Some parameters", file=sys.stderr) + print("textonly_pdf\t1\tSome help text") + sys.exit(0) elif sys.argv[-2] == 'hocr': inputf = sys.argv[-4] output = sys.argv[-3] @@ -63,13 +68,27 @@ def main(): with open(output + '.txt', 'w') as f: f.write('') elif sys.argv[-2] == 'pdf': - inputf = sys.argv[-4] - output = sys.argv[-3] - pdf_bytes = img2pdf.convert([inputf], dpi=300) - with open(output + '.pdf', 'wb') as f: - f.write(pdf_bytes) - with open(output + '.txt', 'w') as f: - f.write('') + if 'textonly_pdf=1' in sys.argv: + inputf = sys.argv[-4] + output = sys.argv[-3] + with Image.open(inputf) as im: + dpi = im.info['dpi'] + imsize = im.size[0] * dpi[0] / 72, im.size[1] * dpi[1] / 72 + + pdf_out = pypdf.PdfFileWriter() + pdf_out.addBlankPage(imsize[0], imsize[1]) + with open(output + '.pdf', 'wb') as f: + pdf_out.write(f) + with open(output + '.txt', 'w') as f: + f.write('') + else: + inputf = sys.argv[-4] + output = sys.argv[-3] + pdf_bytes = img2pdf.convert([inputf], dpi=300) + with open(output + '.pdf', 'wb') as f: + f.write(pdf_bytes) + with open(output + '.txt', 'w') as f: + f.write('') elif sys.argv[-1] == 'stdout': inputf = sys.argv[-2] print("""Orientation: 0 diff --git a/tests/test_main.py b/tests/test_main.py index a5b57f0f..1514e89f 100644 --- a/tests/test_main.py +++ b/tests/test_main.py @@ -10,7 +10,7 @@ import PyPDF2 as pypdf from ocrmypdf.exceptions import ExitCode from ocrmypdf import leptonica from ocrmypdf.pdfa import file_claims_pdfa -from ocrmypdf.exec import ghostscript +from ocrmypdf.exec import ghostscript, tesseract import logging from math import isclose @@ -20,6 +20,11 @@ run_ocrmypdf = pytest.helpers.run_ocrmypdf spoof = pytest.helpers.spoof +RENDERERS = ['hocr', 'tesseract'] +if tesseract.has_textonly_pdf(): + RENDERERS.append('sandwich') + + @pytest.fixture def spoof_tesseract_crash(): return spoof(tesseract='tesseract_crash.py') @@ -210,10 +215,7 @@ def test_high_unicode(spoof_tesseract_noop, resources, no_outpdf): assert p.returncode == ExitCode.bad_args, err -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_oversample(spoof_tesseract_cache, renderer, resources, outpdf): oversampled_pdf = check_ocrmypdf( resources / 'skew.pdf', outpdf, '--oversample', '350', @@ -303,10 +305,7 @@ def test_monochrome_correlation(resources, outdir): assert corr > 0.90 -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_autorotate(spoof_tesseract_cache, renderer, resources, outdir): # cardinal.pdf contains four copies of an image rotated in each cardinal # direction - these ones are "burned in" not tagged with /Rotate @@ -341,10 +340,7 @@ def test_autorotate_threshold( assert eval(correlation_test) -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer',RENDERERS) def test_ocr_timeout(renderer, resources, outpdf): out = check_ocrmypdf(resources / 'skew.pdf', outpdf, '--tesseract-timeout', '1.0') @@ -359,7 +355,7 @@ def test_skip_big(spoof_tesseract_cache, resources, outpdf): assert not pdfinfo[0].has_text -@pytest.mark.parametrize('renderer', ['hocr', 'tesseract']) +@pytest.mark.parametrize('renderer', RENDERERS) @pytest.mark.parametrize('output_type', ['pdf', 'pdfa']) def test_maximum_options(spoof_tesseract_cache, renderer, output_type, resources, outpdf): @@ -515,10 +511,7 @@ def test_tesseract_crash_autorotate(spoof_tesseract_crash, print(err) -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_image_too_big(renderer, spoof_tesseract_big_image_error, resources, outpdf): check_ocrmypdf( @@ -566,10 +559,7 @@ def test_non_square_resolution(renderer, spoof_tesseract_cache, assert in_pageinfo[0].yres == out_pageinfo[0].yres -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract' - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_convert_to_square_resolution(renderer, spoof_tesseract_cache, resources, outpdf): from math import isclose @@ -773,10 +763,7 @@ language_model_penalty_non_freq_dict_word 0 '--tesseract-config', str(cfg_file)) -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_config_notfound(renderer, resources, outdir): cfg_file = outdir / 'nofile.cfg' @@ -788,10 +775,7 @@ def test_tesseract_config_notfound(renderer, resources, outdir): assert p.returncode == ExitCode.ok -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', - ]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_tesseract_config_invalid(renderer, resources, outdir): cfg_file = outdir / 'test.cfg' with cfg_file.open('w') as f: @@ -813,10 +797,7 @@ def test_form_xobject(spoof_tesseract_noop, resources, outpdf): env=spoof_tesseract_noop) -@pytest.mark.parametrize('renderer', [ - 'hocr', - 'tesseract', -]) +@pytest.mark.parametrize('renderer', RENDERERS) def test_pagesize_consistency(renderer, resources, outpdf): from math import isclose diff --git a/tests/test_tess3.py b/tests/test_tess3.py index 0861ebcf..7cef255d 100644 --- a/tests/test_tess3.py +++ b/tests/test_tess3.py @@ -11,14 +11,26 @@ pytestmark = pytest.mark.skipif(tesseract.v4(), reason="tesseract 3.x required") -def test_textonly_pdf_on_tess3(resources, no_outpdf): +@pytest.mark.skipif(tesseract.has_textonly_pdf(), + reason="check that missing dep is reported on old tess3") +def test_textonly_pdf_on_older_tess3(resources, no_outpdf): p, _, _ = pytest.helpers.run_ocrmypdf( resources / 'linn.pdf', - no_outpdf, '--pdf-renderer', 'tess4') + no_outpdf, '--pdf-renderer', 'sandwich') assert p.returncode == ExitCode.missing_dependency +@pytest.mark.skipif(not tesseract.has_textonly_pdf(), + reason="check that feature is exercised on new test3") +def test_textonly_pdf_on_newer_tess3(resources, no_outpdf): + p, _, _ = pytest.helpers.run_ocrmypdf( + resources / 'linn.pdf', + no_outpdf, '--pdf-renderer', 'sandwich') + + assert p.returncode == ExitCode.ok + + def test_oem_on_tess3(resources, no_outpdf): p, _, err = pytest.helpers.run_ocrmypdf( resources / 'aspect.pdf', diff --git a/tests/test_tess4.py b/tests/test_tess4.py index 431b6d9d..56c2b2c4 100644 --- a/tests/test_tess4.py +++ b/tests/test_tess4.py @@ -8,6 +8,7 @@ from ocrmypdf import pdfinfo import sys import os import PyPDF2 as pypdf +from contextlib import contextmanager spoof = pytest.helpers.spoof @@ -32,14 +33,25 @@ def ensure_tess4(): raise EnvironmentError("Can't find Tesseract 4") +@contextmanager +def modified_os_environ(env): + old_env = os.environ.copy() + os.environ = env + yield + os.environ = old_env + + def tess4_available(): """Check if a tesseract 4 binary is available, even if it's not the official "tesseract" on PATH """ try: - ensure_tess4() - return True + # ensure_tess4 locates the tess4 binary we are going to check + env = ensure_tess4() + with modified_os_environ(env): + # Now jump into this environment and make sure it really is Tess4 + return tesseract.v4() and tesseract.has_textonly_pdf() except EnvironmentError: pass