From 8246cc0538f7bf2d436977e6c071bf6dc0204fc6 Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Sat, 20 Feb 2016 04:53:02 -0800 Subject: [PATCH] Gracefully recover from tesseract's failure to process very large images And test cases to check this --- ocrmypdf/tesseract.py | 22 +++++++--- tests/resources/README.rst | 1 + tests/resources/hugemono.pdf | Bin 0 -> 11779 bytes tests/spoof/tesseract_big_image_error.py | 50 +++++++++++++++++++++++ tests/spoof/tesseract_cache.py | 7 ++++ tests/test_main.py | 15 +++++++ 6 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 tests/resources/hugemono.pdf create mode 100755 tests/spoof/tesseract_big_image_error.py diff --git a/ocrmypdf/tesseract.py b/ocrmypdf/tesseract.py index f25d0d3e..020540cc 100644 --- a/ocrmypdf/tesseract.py +++ b/ocrmypdf/tesseract.py @@ -98,6 +98,8 @@ def get_orientation(input_file, language: list, timeout: float, log): return OrientationConfidence(angle=0, confidence=0.0) except CalledProcessError as e: tesseract_log_output(log, e.output, input_file) + if 'Image too large' in e.output: + return OrientationConfidence(0, 0) raise e from e else: osd = {} @@ -148,6 +150,13 @@ def page_timedout(log, input_file): log.warning(prefix + " took too long to OCR - skipping") +def _generate_null_hocr(output_hocr, pageinfo): + with open(output_hocr, 'w', encoding="utf-8") as f: + f.write(HOCR_TEMPLATE.format( + pageinfo['width_pixels'], + pageinfo['height_pixels'])) + + def generate_hocr(input_file, output_hocr, language: list, tessconfig: list, timeout: float, pageinfo_getter, pagesegmode: int, log): @@ -175,13 +184,13 @@ def generate_hocr(input_file, output_hocr, language: list, tessconfig: list, # Temporary workaround to hocrTransform not being able to function if # it does not have a valid hOCR file. page_timedout(log, input_file) - with open(output_hocr, 'w', encoding="utf-8") as f: - pageinfo = pageinfo_getter() - f.write(HOCR_TEMPLATE.format( - pageinfo['width_pixels'], - pageinfo['height_pixels'])) + _generate_null_hocr(output_hocr, pageinfo_getter()) except CalledProcessError as e: tesseract_log_output(log, e.output, input_file) + if 'Image too large' in e.output: + _generate_null_hocr(output_hocr, pageinfo_getter()) + return + raise e from e else: tesseract_log_output(log, stdout, input_file) @@ -242,6 +251,9 @@ def generate_pdf(input_image, skip_pdf, output_pdf, language: list, shutil.copy(skip_pdf, output_pdf) except CalledProcessError as e: tesseract_log_output(log, e.output, input_image) + if 'Image too large' in e.output: + shutil.copy(skip_pdf, output_pdf) + return raise e from e else: tesseract_log_output(log, stdout, input_image) diff --git a/tests/resources/README.rst b/tests/resources/README.rst index da867662..d7ce2f8f 100644 --- a/tests/resources/README.rst +++ b/tests/resources/README.rst @@ -32,6 +32,7 @@ under the terms of the license in LICENSE.rst. - cmyk.pdf (a CMYK image created in Photoshop) - enormous.pdf (a very lage page) - francais.pdf (a page containing French accented characters) +- hugemono.pdf (large monochrome JBIG2 page with pixel dimensions of 35000x35000) - invalid.pdf (a PDF file header followed by EOF marker) - missing_docinfo.pdf (PDF file with no /DocumentInfo section) diff --git a/tests/resources/hugemono.pdf b/tests/resources/hugemono.pdf new file mode 100644 index 0000000000000000000000000000000000000000..30700b3e401a76d444ab73f138cc07dbe5b7efa2 GIT binary patch literal 11779 zcmeHteOOahx^F8GAS8#Vl`p5Vbz1x?Bp@HHq9H=9NHL6Gs3;*7ln_D~B}h@(LV!Bw zoX$|KT4faK0I^U=9)_>Mfi0nZ#%VncO$xy<7}N1g4Ka{u6QBuuzh~{(&dlw-=ehU( zcm8k_vetU{@BLW6^{%zo&a+<5NZA${9Tkl+U;gSZf6_CfF&-v=tALr9h%vc)-Y(9= zxXI!YaiRPeYG;U#<(M%mQ1VuZas(8h5c05$e0hodM83QjOHTH{utRIo#YZGXwDi#0wJ2Kbhj8imf7HdN9bhEOL;(3r`XKgY0tg!bQs z{J+YN$Y__*Sc)9`8I$JsdsE*h2JM6P8z1`v_jJ%dj^x#+k$j(RKXjAH-K}`5#FK!Y zP9c7Gie)Grkv}FkQ<97Nupti7%H(d(lN`%OygzMim6V*w$UB}amley4=YhE@#APXYN94I45M5@7kIOvlbJMWM zXoQj%Dr7}y2GE``?k@Ss@9T)fe5He@f7)M#XpR z^s6{3eZHypVVM9N1Zrr}zvrS6PFVcO8-GI;5s*0j`KOD%|Nnm7(eN_k*gpMdM>3Z! zdUQW$!3(nK~>>)pSx9_oBKll z*sBaZzyH{`oti%_yjv?ie^V0lHG5Ux*}{lVu`ly@_v?6{ufJL=DS9FFoueOqP}<3U zM=HLWcQfwo%et8LNO<;s;81s{}}FP(fY z=O(L@6Bz8x>L>_^NO}IN_bV^$Nb1A#@;B_%GJbZcvGXJ^io;8~_nF}RnT*eol+nnnVB?u}M{b?uj|cZ%4@PB3mHt>6xM(SP1-op+$1-|RKyJC6-9{6a8$ z5{Y}WIQ$_mg?FHL2sY%kz)9rrz1iIqrwl8@aw;Ei60vEGQX6Pj`tg#qfqvmo9fIN2 zFOFbkg6a?f!U^V+Cb?$c>J^+ErCsotul21CH#1PfFOef_uRxUrYyrWlLz2KR5v%aq zr1qf242$;)r2^UHYx8*sjk!fy76yI^8A>FeO=An#gy?Z1hMaHG2JW;9%={5CPw8*= zg~vQ4>081nXGNg0aREaiZ1!!`2I_SSs~f?;QOovKNk!Et@P_IYO9j>7Z|0|W5sh{w z@-c}iN4rPLgbuBku~f^RAsHOJaUw&h_pc5|?iR3VLZxcCX7*JdEDv<((@~Gq7S+74 z$)WeBmTD_FMTE9YU}h{WxuC=S5?9Zs^F9y?t@aOu4wJt*b_7>7lJPp~`t_@U`Z1ke zPsWZ&EIov!5q9bXmX-#K#dT7o*X#WsaBx>S-m=tRRUIyMiTQf_^$*^}KP(WX1Yf6y>>6`6zlAQ=tBUPl-dYrxM8qtlOs|J7dg^w&tQjw%dxFdb(o3KBwsU2Bf@*cZcu6h&{;1G0$f5Ff^Rvut;j9&T5 z{EaKS+KGkA92sijH^RidU$7e!Zm`8)Fgn9l_AI8(NopO*g+Fy9OR}ur_&n$GWt*vNK2^ggJ^19tyrjRha`jgd9 zwR)WMJFD0W*xYvP&k-vV(VXJ@@f(#gwVB~imJ9lX3cm!b;~GERu@J#2TB_Hwts=h! ze~KfU7y4Iwe8_1>G9Aem>YH91jWByz{kBvEYS|l6NIE&Fw)3@Tb_M#yS`zFHsYL3n zG*mdQ*E`TepZ;iI_0&?mWTW3Ell^+Vt7xkJ=FSUbLK9W>FOwDZmcjbeiPj>eP=AJ> zzELPlH#R$rLi;tx^{Zt9qIv-{>5B-J#Z`W0R}sg!$0gLK>zgbIfO2+ovoJmW0xoPa zHBCq=mO7nH)CGh7jITwN`gr3`uwPs4lIzMP7OLJtuI?uj+RA&=doCsj^%nkKy<~Gj z>1BP1Nl&DYN)jBdbTZ!Ra#b{ttJ`+eThT1hox#=Ab)5_8H>e!fx(EWA=JrhTg-zt% zCd=jon7%;Do1FVD;Mo`O^vQ|_lWB4neo(GLx+T|aTotr9X=U2Ef-KK$E`jvX(JdE1 z-{hDCgAf!#S0*&$={=XHNK!X_z8ohiKGMylIs|<%rK7Gn`c)C4wp%96rpE#_LNSgqZ)K>3#h@ywa{8Nv^j7(F3cEaW)#yGh-MJEoijKI`_?z?7@%b=B!q0$F;dMtJ+%8@50q8SE+%Vi>)n^WKLw0Q6#0oD*!Y&Y!A0) z!5H-DV1E9}nnhcBTU-v}8$WN!DHHzPh01XE+jS4$Ao(D4J7K1?ZzSD8%)vryH5sYX zK}@3%?zW&U7dUzhPMhuGryT}z9Z}J+Hj{9}fs2_=OX%JbH00&!&~teX!rKQ32cdeA zTLk1EKpw6_n({OlHqSI8CJ=M&25_TY6S%SzFf(Xy?;rr?+Hr8FQ4OF11$k!9=_bK| zAO=$IBuN~fy$s?fo4Z>`t_~diMHB%f*9lIlz8viHc$&>YfvA7F2MsN50B}9QsX(R4 zg&cb}XvnAqjPPW|&@5rW0mvBme6rO$z%BwG5-0ICsndQswCGT*ZLfPrgK6P2FQEdrTG!uX^WED!a$>;*rco<& z`Yw67)0aK$L|x4;%@2b)g39ECLDqjNo>z-R1_NaP<-^ZU`@Dg|Sg;i?o- z$_!yPAlVGuU-UPql!kxxIcDQR>Qe#;@MNotZu~KtB4J$hkR&-eo zaPN-S@cPj?xLbYJ{uy$(7Bx{!qiSmCN$C08E7)Ar`ntkbDRfQZS372|~YfAkt`Xut-Pm z?@7D{bTpL2?GKg*C~Wjgi4*ta5|Yk9?TU~BRG|o`lqLaH)xb*RX|b}9@aE~uKrQyV zeH$iR;8*!D2X{Alnh2!_^Y7y;Pm(B~eUO<(5&1Y6iCy@@@*z#79KA%$ClOA=0ww~% zB6I@?iv7@tl^uH{Z9)VjCvn4X+zJJCZbj%9Ap4ou@J2qusDWXZaT0?KIEv`piU2E$ zn{LA6%7=pl0_fr41NIHyDi%!;5-R!brvN`j(D&q4v-(!s%4GZ!F%D z63G5^V>X(ck1Nf6XvJ_O+5fzovaIs0{i*?BXFL{{UBqWV|g$c0j!9-C%)V&k{PE_<4#RKbwFEDNVD!9paoq2z=B z${@1g1Qq@-=Pfjm7Ve}=WZ}A-&LcRf2DZ5$W(aVhp1NR0J*OtE4neyK0^eyLKw0S` zT{ue5HUysA0q*I`&C`StnvEdAo1woMiBa4pyu$4?!JLgioC8|Hy4_hYGjKB7X#gk6 zEd%8gAt2gNOiZL1oP>eoid1R`LGA)vWTNWPaGk@GNQ$+0ZD94N8pMD%<+&O-0^?=?n0%4Zey?mQO~M09|wZTeiuMK_G% zroCjpVb(gy;oYoF0k#VCWq_^*O6T4~FC&JTeV3&$OW-1)oM_O2 z5FQl&^w5f`$cpjmuG^>WN62Alekl1 zm?n@JyxZUwjn#uI`_ka#Ns|fMZKEB83ry-6lbf1c4{kTv)qb)qO@@=*-Q~zsIoKlc zuEEjuFf(IxH(QYpX@TXGfmBmHL=XB5T6W`hlR1$wn zAw{EDV6=jjnv}ZFsBM%H%@>10R-yY#SC4`0mdPU4s%H&UyuqC&vT?i4%TeSLwvMp@ zHI%li=SY+7q+2STHO?CKCQ4^ppWV*w>Inm*W=sce#PL(LOW>|B)x6v7&Ne_mfw5aO zFMb{FW3$RRb_-PH@-P=NLnNY%DZzCH)7eUV-Dhr0$1g=k41-=`x9 zSr*7}*?N(zt7js$6wM83o5o;^3Ii*9e*vAwYP+^4cwT&__M-%t)lKZsjy89*BWr^x ztww9yJ2x8y1zO6ukB~J_Hm93FGf*Qd4H0Q>v5<#h=5PQ}VKlj=Zb@>$tnsi$tH#ON zOT#-t_SL7ANs6^F9Kc#HQJ97U@Za+Y4RU3x4ersU-U7Rpc3it;N}I(9pvL#Hf$GFH`{b9{~vG&lb;+N{*fpKwU?Gnu`#*%q)WZm9C5FhZ#coMQ>Fx217R+j^WvzG)LN#V6>2l|2_amrHm#x3Jds)4Vi%o|l=sY_Cc^&l6Fs;MwHDRk_PbpmVu6 zxRcqcO9!$pv%RotK%(@X6a%!8KRO@vPb<`{+m-q_aUGBKOXatgIQ^Cg_4%rKnF@`jH80 zHTr=3xWTb77~d~tD<~~Y)=*k0S!4d*@<7d4*tp2e9M5775aKLc&jfayTio8WWS@xCwH-SI6#VvH&J?jU_}tc z+CTFy_R<)RI6FjCs}X0;i}&+x@(7%@9=B_WVPuB}o%ev2`j~4_@&!{v;;LbPb>Ner zaIy+mn0Db+v3S@&Vi@e`4r8f|5g=u~^5pC&3{~GZ)uNv)d+p+Q=we0CU~)L1cxX2P zj35>^RK_D%K_@egg1PG6NN69gceo{n;YtK;z*vEPM3q#Vx1jPzsad_z?ysU@X zdqc|bd0zUpFIhQ5apr}Ph+*pCP{KSfX{h2SA+Nm3ON|ddckY^f-7idwZt}LSKALnh zc4gJ#Pf`R`i|>81sPWu8iy3~&=ic@5UbX7v{&=PLs+D(Zz4G7L^pj=Y$^U-##z;ZX zx9sGf@}v>dVFwOldH-1#68Ck%@b=E<(Ql1vhDes zz1u@o`Rr9DWqi_}FPC3>IP|+iXYZv?vHh2QQhB=bc?Q~W#0G8EVD(|O$NL&N?`2p1 zy8d0piI=>5`s+BkdqO||c`;^7%8%4nE?)eTJ$q8#Sd1>vIR-LChtJ@aauJn&=l!mb z%grm}y_+I#pU)P&tGpAfsp#~X>qr`Ob~KK$rRb%}!j(!5q@ttSWo-zddV1R0cIaCB`#*kJa!{}iEm7-qvIyOdppgCIt#<-{o1T-ZxgcKk!LR*u zlkD0I?Jn%}ZiYWE=l#78hjNBQZ=#8h=6~i}AE?vg4z_ma54T*EqMD@D7H6r}wlkD_ zB`sG+#EfMUy2TOiQwmzOEKbgc2N@jRRZmUq<;{9wjW}rQ!`1^|R_@_7`QBIg_|*Bu z1#BEIctM-7rY`*Gl-~iSQhYQc{)M|kt5)&RbL`pmjDcqiJY(P)1J4+E#=tWMo-y!@ zfoBXnW8fJB&lvb0&A@H%=aT;SBc$}0$McRddDv?hlRj7b^uprf z;x@!#NB<#