From a5efc4af9bc5d172ed841fa54eaa10cb8f3358f1 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sun, 3 Jul 2022 15:21:58 -0700 Subject: [PATCH] unpaper: replace input pnm with png Unpaper or its underlying libraries don't seem to accept pnms with an odd integer width. Although it's not clear if this is the issue at all. In any case, keeping the image a PNG works around the issue. unpaper only accepted PNM input in the past, which is why we send it PNM. Since it now accepts PNG, we might as well use PNG. Unpaper can write PNG as output too, but this added a few seconds to the test suite was not committed. Related issues: https://github.com/ocrmypdf/OCRmyPDF/issues/887 https://github.com/ocrmypdf/OCRmyPDF/issues/665 https://github.com/unpaper/unpaper/issues/82 --- src/ocrmypdf/_exec/unpaper.py | 38 +++++++++--------- .../pdf.bin | Bin 0 -> 3626 bytes .../stderr.bin | 1 + .../stdout.bin | 0 .../txt.bin | 13 ++++++ tests/cache/manifest.jsonl | 1 + 6 files changed, 34 insertions(+), 19 deletions(-) create mode 100644 tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin create mode 100644 tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin create mode 100644 tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin create mode 100644 tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin diff --git a/src/ocrmypdf/_exec/unpaper.py b/src/ocrmypdf/_exec/unpaper.py index 479959ef..aad2a9b4 100644 --- a/src/ocrmypdf/_exec/unpaper.py +++ b/src/ocrmypdf/_exec/unpaper.py @@ -72,13 +72,13 @@ def version() -> str: return get_version('unpaper') -SUFFIXES = {'1': '.pbm', 'L': '.pgm', 'RGB': '.ppm'} +SUPPORTED_MODES = {'1', 'L', 'RGB'} -def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]: +def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool]: im_modified = False - if im.mode not in SUFFIXES: + if im.mode not in SUPPORTED_MODES: log.info("Converting image to other colorspace") try: if im.mode == 'P' and len(im.getcolors()) == 2: @@ -91,13 +91,11 @@ def _convert_image(im: Image.Image) -> Tuple[Image.Image, bool, str]: ) from e else: im_modified = True - try: - suffix = SUFFIXES[im.mode] - except KeyError: - raise MissingDependencyError( - "Failed to convert image to a supported format." - ) from None - return im, im_modified, suffix + if im.mode not in SUPPORTED_MODES: + raise MissingDependencyError( + "Failed to convert image to a supported format." + ) from None + return im, im_modified @contextmanager @@ -105,19 +103,21 @@ def _setup_unpaper_io(input_file: Path) -> Iterator[Tuple[Path, Path, Path]]: with Image.open(input_file) as im: if im.width * im.height >= UNPAPER_IMAGE_PIXEL_LIMIT: raise UnpaperImageTooLargeError(w=im.width, h=im.height) - im, im_modified, suffix = _convert_image(im) + im, im_modified = _convert_image(im) with TemporaryDirectory(ignore_cleanup_errors=True) as tmpdir: tmppath = Path(tmpdir) - if im_modified or input_file.suffix != '.pnm': - input_pnm = tmppath / 'input.pnm' - im.save(input_pnm, format='PPM') + if im_modified or input_file.suffix != '.png': + input_png = tmppath / 'input.png' + im.save(input_png, format='PNG') else: # No changes, PNG input, just use the file we already have - input_pnm = input_file + input_png = input_file - output_pnm = tmppath / f'output{suffix}' - yield input_pnm, output_pnm, tmppath + # unpaper can write .png too, but it seems to write them slowly + # adds a few seconds to test suite - so just use pnm + output_pnm = tmppath / 'output.pnm' + yield input_png, output_pnm, tmppath def run_unpaper( @@ -125,7 +125,7 @@ def run_unpaper( ) -> None: args_unpaper = ['unpaper', '-v', '--dpi', str(round(dpi, 6))] + mode_args - with _setup_unpaper_io(input_file) as (input_pnm, output_pnm, tmpdir): + with _setup_unpaper_io(input_file) as (input_png, output_pnm, tmpdir): # To prevent any shenanigans from accepting arbitrary parameters in # --unpaper-args, we: # 1) run with cwd set to a tmpdir with only unpaper's files @@ -133,7 +133,7 @@ def run_unpaper( # 3) append absolute paths for the input and output file # This should ensure that a user cannot clobber some other file with # their unpaper arguments (whether intentionally or otherwise) - args_unpaper.extend([os.fspath(input_pnm), os.fspath(output_pnm)]) + args_unpaper.extend([os.fspath(input_png), os.fspath(output_pnm)]) run( args_unpaper, close_fds=True, diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/pdf.bin new file mode 100644 index 0000000000000000000000000000000000000000..937ee88545e568a8d347d0bd38d1ec87ee9b2b16 GIT binary patch literal 3626 zcmbVP30M?I7H%$;!H9f{;x4-7fud%hdkzo-$T5JUAUGfd5v7^wnQ4XYVWwGUFhLYo zlh5F4lxTuSG(ch?QAD$9_`Kt}D&~u8jCdtt+||S@d>ZlCS3NTfhoAX&3#Pl?tM{s2 zy?Xzv*FkBjL@5=blmuP*V&82Eg-FEb&y_?*N)QUou$dSX8cz#!0dK&JG}^#ekPL)0 zlIUm&!|AZFyo*vCZDA65PC%ha1vZm0nXy|yf}0lp8ucDFaP5@FNppSVUE2y;uA-S-8CrG;TXA6 zj?cj%7Ye>sb1Z~xnwi(8GXm1HoX*Tx_+qn`LHTg3oP?4g9jg@_l*rQ-(k2O3?i46? zpV5_;ef_5ZavrBRU%C7(Gk}Q8b-X;YtmxLG%U=e6gc9ZbA zT1>Q-G1HuZkwlWvqETcbG!d!Y#vNMu`j?0!yE4YNa~!(jZab{g7Mslw(V|O4o3Sk{ z$8>nj#GCL0&R^Ff!Qt+T)>xbt1Vn+w*e&t2X)?naj4qBnaR^9CiO?ya4Hl#nyMU0m zINpjNmyqxWm=LG2rxc7Ih<-+fCcv1+sbKeRkL;v|G{;xsfJz@LdG*mF)t zf^mVi6Z-J{uKhTIOc^c$*G^FgldhN}FwP`I8R0>o)bvnU=9S}X2*NEGxOv#qFs1-{ zCD6kR1vb6^h}cm;?*=-+$j~|;>wO0L_3#Tb0;8|Hr#tj-z;BeXP_TMJ2jt&@N1p;- zOA~!XdOJLO71CA{;YoPGcpsn<2jTYM_MjhtZU%a)iMI$~A6V!fh{tgc|F*9w>OaRf zvdyzMRwIbBA3f#p`@6%RKEQ8JLge=+uqC*GP9Fe19>EQsc&5(HUF5jKkbR{4I062_ zUI1HP-n>AurxKP;N=;27Vni)BYy%c76UmN;L(y&>SZ${V;x(ZSBznOqbZ*#(qKvpx zkfMhB4fiCT+V|~2jDvBk>qQK5NSO}Kj364MnMT8J!CS$g+ zJcpF*q2C(W1r-&}$XTK@%U5z6?jm9{2T-rdVFgljTkjcS-kx^kGfEoKs zyocifG@xJs9KpB<8jT|pT3|lsHHhy!XSU3Nt|}3(D8>poR9y%w$2s0=$2(p`B815p!{vOm7hDl?<_7P9ZfXZ>wWxs-s4|mG&g?w>60rd_l$vW-+0!3`swaS>009l z`;YPqZ64?HspvO+^fvjN*?(y%m!E%qTdgXmDf{_{n{L)wPDCsVJ<}TF+dTit$Vn3i zw}d}D_4?J6Ssoz^`;M-%u%!X>FX?|i%w;{zTf6GcxMR7Sw!9U$s3L_sgoNU-j!gm4)0qxPV_Zn||o#LtOrQ-l3m| z%9_r3>YzO(m6LX~7Eh_yCZr6_Pf84XufZp4Lt}+sFUyI2 zRRi7$jH)=d*?dqw`Akml#hQ??CC$q}TktgTGjbI&9WeM0EID~?X=&K%ruToizi5X4 zt{wNi+7}H~@*~UFdlV=6S;N{IipG`Kq+J$L& zxH~JGrv;1;9Q^stm?N=c>#G~cYhf+51LJ2ET}>C3ow=Mh>G~S0BAeLcAK^1U`SIw{ z+&3%y7Rjprx@JjUWAKRKLn>EHR8D#CQrVFEH~ZC|3iQjZ`_KF9N8J0de*CB=Q{cB{ zhgY5qSoG(I){RGPty6Ow^}^gC=yLO;sLd78+0Ukp2~5r{zZ+n!-lspGJmj(BTw;T? z&tl`463MCS>o?YoViOw=tf)Tm0ixebJAWX^x1oL5W-iVh}5dzi(tqFYV{suYKNoyb>8_{`O#Kb?Vw%Q@<`by>{h*4c=Kd+h)A>jUvF4%UZs5 zXdG9xe}f<&(eSZ9H7ona@1_^?$phN-|E_7zKYCU@d)K2sojLCPy?OA~G5;7=^X;bo z2lX@iJvv#gsUKMV_Qh=zDpyF3RqUxyjB9ODmt8Xd?t8C!hqOn&!CwF_aJuphP7t9} zSsgCzaZYlS_VIi%zTP{-yX%JrBobXQR4ynu13nj^f`K|q5kiGfVwn^Vg{lC#Q3drY z3RX>&kun(>PRc1IIYFtAlHp@Wa*QaM!3uDBMapfHe0a8YD$C`_r8 zE0G>`@F*yuaYLMwC#PgkJ9g!fWEfob-FRS651v8^MNT)la77O}ic&xh>?%jemEfOl zyoiV%y0B~yo}7y4q$`+dsDI3k@40lgguz~bL(=d(oQN~Q<4+t!*-5cPq0NHmG`XCl MBtbz5sfm*R0@R`*asU7T literal 0 HcmV?d00001 diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin new file mode 100644 index 00000000..16b617e5 --- /dev/null +++ b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stderr.bin @@ -0,0 +1 @@ +Tesseract Open Source OCR Engine v4.1.1 with Leptonica diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/stdout.bin new file mode 100644 index 00000000..e69de29b diff --git a/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin new file mode 100644 index 00000000..25fdded2 --- /dev/null +++ b/tests/cache/francais/__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt/txt.bin @@ -0,0 +1,13 @@ +Portez ce vieux whisky au juge +blond qui fume sur son Ile +interieure, a cöte de l'alcöve +ovoide, oU les büches se +consument dans l'ätre, ce qui +lui permet de penser & la +caenogenese de |'etre dont il +est question dans la cause +ambigu& entendue a MoY, dans +un capharnaüm qui, pense-t-il, +diminue ca et la la qualite de son +ceuvre. + \ No newline at end of file diff --git a/tests/cache/manifest.jsonl b/tests/cache/manifest.jsonl index fdd1affe..995ec062 100644 --- a/tests/cache/manifest.jsonl +++ b/tests/cache/manifest.jsonl @@ -82,3 +82,4 @@ {"tesseract_version": "4.1.1", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__--oem__1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "--oem", "1", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} {"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=1__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} {"tesseract_version": "5.0.0", "system": "Linux", "python": "3.9.5", "argv_slug": "__-l__eng__thresholding_method=2__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/trivial.pdf", "args": ["-l", "eng", "-c", "textonly_pdf=1", "-c", "thresholding_method=2", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]} +{"tesseract_version": "4.1.1", "system": "Linux", "python": "3.10.4", "argv_slug": "__-l__deu__000001_ocr.png__000001_ocr_tess__pdf__txt", "sourcefile": "resources/francais.pdf", "args": ["-l", "deu", "-c", "textonly_pdf=1", "$TMPDIR/000001_ocr.png", "$TMPDIR/000001_ocr_tess", "pdf", "txt"]}