From 795019b0c1de8964d6a7cd279d08cdf3fb37b2ab Mon Sep 17 00:00:00 2001 From: "James R. Barlow" Date: Tue, 11 Sep 2018 14:44:16 -0700 Subject: [PATCH] Work around invalid TOC entries Kodak Capture Desktop and probably other software creates a /Outlines entry with /First being set to an invalid indirect reference to an object that hasn't been created. This is legal in the PDF spec but problematic for qpdf. The objgen will be (max valid object ID + 1, 0). Because we create new objects in _weave, some TOC entries will end up assigned to new objects we create. Typically /ProcSet. We solve the issue by refactoring page traversal and then doing it twice, once to resolve all references (eliminating the null reference problem) and a second pass to make our changes. --- src/ocrmypdf/_weave.py | 91 ++++++++++++++++++++++++++----------- tests/resources/README.rst | 5 ++ tests/resources/kcs.pdf | Bin 0 -> 4372 bytes tests/test_metadata.py | 12 +++++ 4 files changed, 81 insertions(+), 27 deletions(-) create mode 100755 tests/resources/kcs.pdf diff --git a/src/ocrmypdf/_weave.py b/src/ocrmypdf/_weave.py index 450e6417..c4c20b79 100644 --- a/src/ocrmypdf/_weave.py +++ b/src/ocrmypdf/_weave.py @@ -137,14 +137,9 @@ def _find_font(text, pdf_base): return font, font_key -def _fix_toc(pdf_base, pageref_remap, log): - """Repair the table of contents - - Whenever we replace a page wholesale, it gets assigned a new objgen number - and other references to it within the PDF become invalid, most notably in - the table of contents (/Outlines in PDF-speak). In weave_layers we collect - pageref_remap, a mapping that describes the new objgen number given an old - one. (objgen is a tuple, and the gen is almost always zero.) +def _traverse_toc(pdf_base, visitor_fn, log): + """ + Walk the table of contents, calling visitor_fn() at each node The /Outlines data structure is a messy data structure, but rather than navigating hierarchically we just track unique nodes. Enqueue nodes when @@ -152,8 +147,6 @@ def _fix_toc(pdf_base, pageref_remap, log): the two types of object in the table of contents that can be page bookmarks and update the page entry. - It may ultimately be better to find a way to rebuild a page in place. - """ visited = set() @@ -162,10 +155,54 @@ def _fix_toc(pdf_base, pageref_remap, log): if not '/Outlines' in pdf_base.root: return + + queue.add(pdf_base.root.Outlines.objgen) + while queue: + objgen = queue.pop() + visited.add(objgen) + node = pdf_base.get_object(objgen) + log.debug('fix toc: exploring outline entries at %r', objgen) + + # Enumerate other nodes we could visit from here + for key in link_keys: + if key not in node: + continue + item = node[key] + if not item.is_indirect: + # or not isinstance(item, pikepdf.Dictionary): + # # If there is garbage data, replace the key with an indirect + # # ref to None. Kodak Capture Desktop produces keys like these. + # log.error('Removing invalid reference from TOC: %s', repr(item)) + # node[key] = pdf_base.make_indirect(None) + continue + objgen = item.objgen + if objgen not in visited: + queue.add(objgen) + + visitor_fn(pdf_base, node, log) + + +def _fix_toc(pdf_base, pageref_remap, log): + """Repair the table of contents + + Whenever we replace a page wholesale, it gets assigned a new objgen number + and other references to it within the PDF become invalid, most notably in + the table of contents (/Outlines in PDF-speak). In weave_layers we collect + pageref_remap, a mapping that describes the new objgen number given an old + one. (objgen is a tuple, and the gen is almost always zero.) + + It may ultimately be better to find a way to rebuild a page in place. + + """ + if not pageref_remap: return def remap_dest(dest_node): + """ + Inner helper function: change the objgen for any page from the old we + invalidated to its new one. + """ if not isinstance(dest_node, pikepdf.Array): return pageref = dest_node[0] @@ -174,30 +211,24 @@ def _fix_toc(pdf_base, pageref_remap, log): new_objgen = pageref_remap[pageref.objgen] dest_node[0] = pdf_base.get_object(new_objgen) - queue.add(pdf_base.root.Outlines.objgen) - while queue: - objgen = queue.pop() - visited.add(objgen) - node = pdf_base.get_object(objgen) - log.debug('fix toc: visiting %r', objgen) - - # Enumerate other nodes we could visit from here - for key in link_keys: - if key not in node: - continue - item = node[key] - if not item.is_indirect: - continue - objgen = item.objgen - if objgen not in visited: - queue.add(objgen) + def visit_remap_dest(pdf_base, node, log): + """ + Visitor function to fix ToC entries + Test for the two types of references to pages that can occur in ToCs. + Both types have the same final format (an indirect reference to the + target page). + """ if '/Dest' in node: + # /Dest reference to another page (old method) remap_dest(node['/Dest']) elif '/A' in node: + # /A (action) command set to "GoTo" (newer method) if '/S' in node['/A'] and node['/A']['/S'] == '/GoTo': remap_dest(node['/A']['/D']) + _traverse_toc(pdf_base, visit_remap_dest, log) + def weave_layers( infiles, @@ -247,6 +278,12 @@ def weave_layers( pdfinfo = context.get_pdfinfo() pagerefs = {} + # Walk the table of contents first, to trigger pikepdf/qpdf to resolve all + # page references in the table of contents. Some PDF generators put invalid + # references in the ToC, so we want to resolve them to null before we + # create any references, or the ToC will be corrupted + _traverse_toc(pdf_base, lambda *args: None, log) + procset = pdf_base.make_indirect( pikepdf.Object.parse(b'[ /PDF /Text /ImageB /ImageC /ImageI ]')) diff --git a/tests/resources/README.rst b/tests/resources/README.rst index cc2caa8a..63fdd705 100644 --- a/tests/resources/README.rst +++ b/tests/resources/README.rst @@ -104,6 +104,11 @@ licensed under the specified license. - @jbarlow83 - @jbarlow83 - CC-BY-SA 4.0 + * - kcs.pdf + - PDF file generated by Kodak Capture Desktop Software 1.2; has invalid table of contents + - @jbarlow83 + - @jbarlow83 + - CC-BY-SA 4.0 * - masks.pdf - file containing explicit masks and a stencil mask drawn without a proper transformation matrix; printout of a German Wikipedia article (CC-BY-SA) - @supergrobi diff --git a/tests/resources/kcs.pdf b/tests/resources/kcs.pdf new file mode 100755 index 0000000000000000000000000000000000000000..b0d46990e5839d3a82f6f744fbcf79f893ecd302 GIT binary patch literal 4372 zcmZXY2V9fq_Q$W1giMG8WVl{7S40AYA*dt-EtHj_0#`wKB>@5%2q9tsX@-GTYY_)n zs|_m1kZ~!BGDH$>siPNzvZ7)-Ze>K$HXvmr|2Gu-?`@xaK6%zTzjL1R;H2@=0Y9S!WRSzlW`Jz;A+%A(C})&1 z_9<-iQc9s*^ap68&a&mQqS4=|=;QS+hB5c$7{8x=`D|#WaBfm&bPl?c;_C=;6r=fe)kLaC7tpe zO?y!r!4-!7wO0?s4mmKJraVl5U$22Pxy@m1P78IJ zqZD(=cs*IvK8F+2UJkfl4UXtIrqLF(pG|(<8?JaTJ8!#e_AH+Enj9X%CL~z*SFc% zxqbZOldPx&vRtzM909g&CH@HFp0(fp z2{f_f5~?4K7es$%My@<$4kYtNG6*!v-m+Rmi93aQiJazCz_Cl;1{;%vuIu&d4HKzj zActGdB8h!(MlXw`8_Bq2!P?xj5|?!$n$nmhR^BK+hMH)l$O6awN>Vp)2+=}6aj5qM z^VL}ibz>b^dga@1?w0W{Q5x+~gOshn%+A%6gsAwa%eL3-Kc-OzHufb-ZLy89I&$_Za2M!zyg4V*NXt=OyFZCVSPrV*wAUO)|A2IAlJxDF zs#YrVLA}NkAqvmW7@t-l&UO@4qD4tv?aEtOV{AisT(`arQ9fQ4wh`Wu-&oG_4s@K$ zFXFvFj=8D7bZO|gXZ#`)79xNf7Wuutn+A1tO|XvqxlZ9>T!HsGiz3-HUv-CX}WJ(s}4@OJR9R;y2n?=%&fH}8T3s@M;p{q1?vR-z*t^V_f)0| z8$c|vSsOgayQ6O(SJ^kd`Dbxc1Z@vz#?U4GN$l&g!5~)^{oH6nRFF)N-&a%p6Ar{D zcN8(7b+E)4E3MhF(?jlUMI%Fgwzy`wM3L1V(SpH^Yo2v15QpJg%_{iax9FA^%HyVP zOtHLh5(Vw7Qe8U2Ieqj((zZ5cF@+hA$g{6_9>E@%>Le=I{`BiU)_V!$C-XxY6F1jM z)+F~<_|A210AG&qFG#VymaD0W(miMQc5R<$*Jq4P`X`I;#UE2?Mrd+@Io1b|`6CGX zGPDmFDB0MjK+lF&Tz1}7@izr4^3bgYqUr@yV+)we5lWBcF-@KDpO1I0v zJ;9l2Zk~J;s4a0j-2CRb_=7mC)+a-u5L403XM|bjs8M(7REX0Sc8Z%yy4}Y^tgW8# z(x*F02RVf|DK{LLZ_eYq^1&L{9yeaEx9nliRAur0P>v(<1K3nYcIvnHj^y6Fzl(3d`Ml{!pyMytEpT#k-3a&Mn8abdDry(j z-Q3K9RN!lZ{(`%BvkyE(Y0nHy?Lq!L+z{QD_tB-q4rar(tQnVzPm=rc#rvdUlvZa~ zDfw=_qq-W;td|XQw_Rzu2d=sgrc~ogq}D@GapKd- zUO}zFSkF%meEIe9{TAv&27?lMjE%lH+HIIH)cAtbpkU}}_iN{SDM!-k9XQqtKFxV2 z+&9uf?m38URhkUr;?5r~RWYCZU^TKcy2OHXHqTq> z9-5`vQWJx@bmEq;9cr|k z;jJNuHY@&1tz0#q2XIw=MFoU6s;27cdHxq|3ys$Lc6Sj<^%gcidQD}}+^IDfx((cE zgMnjcGt5lPbs0wI42#vHi}Uq`Q%CP~MV?tz)ILA2F<`f!949o%RYBT5ewM!TFwFKj z;y?30rOK3XJy(A1+Oa5kP?PbSr>(_$e%66E1k{gFuS=>!CM~7+zxt^F9r)@qJ_YvF zg&%8M0gR5&uUpl)UZlcm{MHFgQcjaIr$mjY|0X{t1kup#MOn1Q+gXoL7oQ^Liuz+P zc36V^u(l%6@;Xq7XAjrQc@dY@qQFxJU98dP-Ax9cU|V(6jW;B<#ETBLV)?18 zZHp@f*iBAl^h9&`GrU(=q_gfKe&|&C#>iE09~D`gxnG$_m6g#-yvKF;+=$oGo`wMO zRdY#+&AQ9DI-M{u+f+I!y(20@TEK*X_H`s3K?&Ej+R3hBOkt($l}B%sCeSbHnx6xe zXk;MpG*$!G?&XQ%tAMMk3F*>gm$IvrI!n=)z~fkqX;qlJsBkkiut3@m#K=wyP)Z&_ zfa#eKQ0R92Ov*O@{pGk?TmYt>e8!g&hTnPBO@~n?fM;mPqO;&RMeBLD zPSJYN5O_I&L@Y`G6pL>m_0eBU*Ic+WuXFQ%k$iF*7coC50~>#X7nb7|-#+!%Rj#{= zb1O0+*9f$fR&qyin?brExAfUiN9jRfSH~Q3s;5>EDUR!*P8rk)eS^Pj${@i8GsCQh zDye6@ROX;T&eqG3`|{1Z>!FP*nWDh|&LCO5;9R{%)zS2W72idFw)#aID|m()�V_ zvtxIdn%PY?$LKn6T21#JI!42895-9Tf1#MWd7)6!I7;^`YqPbnlhndlZeprN+x(wcVbmyg;(?SDv|84%lKA@TMakJ{ZV;Tb{gLf zyw38FJQ-eb#%#a}L3r=r3h5Wtz-kAqqeI{0 z0hiDZzxsD`QhoC!`S;M*@!TDQK{Y*N&`s&@bPrV4qaOvh86o5cd{*)ms6W%|`Te=d z&qhBrd~@T#(uSq|KC|6uI^w`Ah&YVFx*bK$mPm^~Ff3dUy$m*-`PySV81gef^T&C{ z{>~6l5!8(A$K<3L_aTG-M^o=Vw3m^Y3i3b}H_b@RX^fnA z&>WM?V=+0<+(@ahfATld>i=NplADnVbAQLf2qrxTLS-YG$D)(|k|K@gU-ulq@szCHJbQG1N?k_eSCpbV0q0afCyFA*Z|HS zYg?e%-`4z~O#jyonnAh#WzEYEx|P4Id6S@O{B`Z`c0T?Af6IkL^oDNYoi8sNWU?69 z(Ek_^y)y+2