v7.3.1 release notes

.readthedocs.yml

@@ -0,0 +1,10 @@
+build:
+  image: latest
+
+python:
+  version: 3.6
+
+formats:
+  - pdf
+
+requirements_file: requirements/main.txt

README.md

@@ -1,7 +1,7 @@
 OCRmyPDF
 ========
 
-[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew]
+[![Travis build status][travis]](https://travis-ci.org/jbarlow83/OCRmyPDF) [![PyPI version][pypi]](https://pypi.org/project/ocrmypdf/) ![Homebrew version][homebrew] ![ReadTheDocs][docs]
 
 [travis]: https://travis-ci.org/jbarlow83/OCRmyPDF.svg?branch=master "Travis build status"
 
@@ -9,6 +9,8 @@ OCRmyPDF
 
 [homebrew]: https://img.shields.io/homebrew/v/ocrmypdf.svg "Homebrew version"
 
+[docs]: https://readthedocs.org/projects/ocrmypdf/badge/?version=latest "RTD"
+
 OCRmyPDF adds an OCR text layer to scanned PDF files, allowing them to be searched or copy-pasted.
 
 ```bash

docs/conf.py

@@ -74,7 +74,7 @@ if on_rtd:
         def __getattr__(cls, name):
             return MagicMock()
 
-    MOCK_MODULES = ['pikepdf', 'libxmp', 'libxmp.utils']
+    MOCK_MODULES = ['pikepdf', 'libxmp', 'libxmp.utils', 'ocrmypdf.leptonica']
     sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES)
 
 

docs/release_notes.rst

@@ -14,6 +14,16 @@ Note that it is licensed under GPLv3, so scripts that ``import ocrmypdf`` and ar
    replace: `#$1 <https://github.com/jbarlow83/OCRmyPDF/issues/$1>`_
 
 
+v7.3.1
+------
+
+-   Fixed performance regression from v7.3.0; fast page analysis was not selected when it should be.
+
+-   Fixed a few exceptions related to the new ``--mask-barcodes`` feature and improved argument checking
+
+-   Added missing detection of TrueType fonts that lack a Unicode mapping
+
+
 v7.3.0
 ------
 

src/ocrmypdf/__main__.py

@@ -545,6 +545,11 @@ def check_options_ocr_behavior(options, log):
         raise argparse.ArgumentError(
             None,
             "Error: choose only one of --force-ocr, --skip-text, --redo-ocr.")
+    if options.force_ocr and any((options.mask_barcodes, options.threshold)):
+        raise argparse.ArgumentError(
+            '--force-ocr',
+            'Error: --force-ocr currently may not be used with --threshold or --mask-barcodes'
+        )
 
 
 def check_options_optimizing(options, log):

src/ocrmypdf/_pipeline.py

@@ -167,7 +167,7 @@ def repair_and_parse_pdf(
         detailed_page_analysis = True
 
     try:
-        pdfinfo = PdfInfo(output_file, log=log)
+        pdfinfo = PdfInfo(output_file, detailed_page_analysis=detailed_page_analysis, log=log)
     except pikepdf.PasswordError as e:
         raise EncryptedPdfError()
     except pikepdf.PdfError as e:
@@ -621,7 +621,7 @@ def select_ocr_image(
             # Calculate resolution based on the image size and page dimensions
             # without regard whatever resolution is in pageinfo (may differ or
             # be None)
-            bbox = textarea
+            bbox = [float(v) for v in textarea]
             xscale, yscale = float(xres) / 72.0, float(yres) / 72.0
             pixcoords = [bbox[0] * xscale,
                          im.height - bbox[3] * yscale,
@@ -631,7 +631,6 @@ def select_ocr_image(
             log.debug('blanking %r', pixcoords)
             draw.rectangle(pixcoords, fill=white)
             #draw.rectangle(pixcoords, outline=pink)
-        del draw
 
         if options.mask_barcodes or options.threshold:
             pix = leptonica.Pix.frompil(im)
@@ -645,6 +644,7 @@ def select_ocr_image(
                     draw.rectangle(rect, fill=white)
             im = pix.topil()
 
+        del draw
         # Pillow requires integer DPI
         dpi = round(xres), round(yres)
         im.save(output_file, dpi=dpi)

src/ocrmypdf/leptonica.py

@@ -21,6 +21,7 @@
 # Python FFI wrapper for Leptonica library
 
 from collections.abc import Sequence
+from contextlib import suppress
 from ctypes.util import find_library
 from functools import lru_cache
 from io import BytesIO
@@ -36,12 +37,12 @@ from .helpers import fspath
 
 # pylint: disable=protected-access
 
-lept = ffi.dlopen(find_library('lept'))
-
 logger = logging.getLogger(__name__)
 
+lept = ffi.dlopen(find_library('lept'))
 lept.setMsgSeverity(lept.L_SEVERITY_WARNING)
 
+
 def stderr(*objs):
     """Shorthand print to stderr."""
     print("leptonica.py:", *objs, file=sys.stderr)
@@ -410,8 +411,9 @@ class Pix(LeptonicaObject):
             smoothx, smoothy = kernel_size
             p_pix = ffi.new('PIX **')
 
+            pix = Pix(lept.pixConvertTo8(self._cdata, 0))
             result = lept.pixOtsuAdaptiveThreshold(
-                self._cdata,
+                pix._cdata,
                 sx, sy,
                 smoothx, smoothy,
                 scorefract,
@@ -432,8 +434,9 @@ class Pix(LeptonicaObject):
             if isinstance(mask, Pix):
                 mask = mask._cdata
 
+            pix = Pix(lept.pixConvertTo8(self._cdata, 0))
             thresh_pix = lept.pixOtsuThreshOnBackgroundNorm(
-                self._cdata,
+                pix._cdata,
                 mask,
                 sx, sy,
                 thresh, mincount, bgval,
@@ -560,22 +563,28 @@ class Pix(LeptonicaObject):
         return Pix(lept.pixInvert(ffi.NULL, self._cdata))
 
     def locate_barcodes(self):
-        with _LeptonicaErrorTrap():
+        try:
-            pix = Pix(lept.pixConvertTo8(self._cdata, 0))
+            with _LeptonicaErrorTrap():
-            pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0))
+                pix = Pix(lept.pixConvertTo8(self._cdata, 0))
-            sarray = StringArray(lept.pixReadBarcodes(pixa_candidates._cdata,
+                pixa_candidates = PixArray(lept.pixExtractBarcodes(pix._cdata, 0))
-                                                    lept.L_BF_ANY,
+                sarray = StringArray(lept.pixReadBarcodes(
-                                                    lept.L_USE_WIDTHS,
+                    pixa_candidates._cdata,
-                                                    ffi.NULL,
+                    lept.L_BF_ANY,
-                                                    0))
+                    lept.L_USE_WIDTHS,
-            for n, s in enumerate(sarray):
+                    ffi.NULL,
-                decoded = s.decode()
+                    0
-                if s.strip() == '':
+                ))
-                    continue
+        except (LeptonicaError, ValueError) as e:
-                box = pixa_candidates.get_box(n)
+            return
-                left, top = box.x, box.y
+
-                right, bottom = box.x + box.w, box.y + box.h
+        for n, s in enumerate(sarray):
-                yield (decoded, (left, top, right, bottom))
+            decoded = s.decode()
+            if decoded.strip() == '':
+                continue
+            box = pixa_candidates.get_box(n)
+            left, top = box.x, box.y
+            right, bottom = box.x + box.w, box.y + box.h
+            yield (decoded, (left, top, right, bottom))
 
     def despeckle(self, size):
         if size == 2:
@@ -740,6 +749,7 @@ class Sel(LeptonicaObject):
 
     @classmethod
     def from_selstr(cls, selstr, name):
+        # TODO this will strip a horizontal line of don't care's
         lines = [line.strip() for line in selstr.split('\n') if line.strip()]
         h = len(lines)
         w = len(lines[0])

src/ocrmypdf/pdfinfo/__init__.py

@@ -588,12 +588,12 @@ def _pdf_get_pageinfo(pdf, pageno: int, infile, xmltext):
     return pageinfo
 
 
-def _pdf_get_all_pageinfo(infile, detailed_page_analysis, log=None):
+def _pdf_get_all_pageinfo(infile, detailed_analysis=False, log=None):
     if not log:
         log = Mock()
 
     pdf = pikepdf.open(infile)
-    if not detailed_page_analysis:
+    if detailed_analysis:
         pages_xml = None
     else:
         pages_xml = ghosttext.extract_text_xml(infile, pdf, pageno=None, log=log)
@@ -601,17 +601,18 @@ def _pdf_get_all_pageinfo(infile, detailed_page_analysis, log=None):
     pages = []
     for n in range(len(pdf.pages)):
         page_xml = pages_xml[n] if pages_xml else None
-        page = PageInfo(pdf, n, infile, page_xml)
+        page = PageInfo(pdf, n, infile, page_xml, detailed_analysis)
         pages.append(page)
 
     return pages, pdf
 
 
 class PageInfo:
-    def __init__(self, pdf, pageno, infile, xmltext):
+    def __init__(self, pdf, pageno, infile, xmltext, detailed_analysis=False):
         self._pageno = pageno
         self._infile = infile
         self._pageinfo = _pdf_get_pageinfo(pdf, pageno, infile, xmltext)
+        self._detailed_analysis = detailed_analysis
 
     @property
     def pageno(self):
@@ -623,6 +624,8 @@ class PageInfo:
 
     @property
     def has_corrupt_text(self):
+        if not self._detailed_analysis:
+            raise NotImplementedError('Did not do detailed analysis')
         return any(tbox.is_corrupt for tbox in self._pageinfo['textboxes'])
 
     @property

src/ocrmypdf/pdfinfo/layout.py

@@ -29,7 +29,7 @@ from pdfminer.layout import (LAParams, LTChar, LTContainer, LTLayoutContainer,
                              LTPage, LTTextBox, LTTextLine)
 from pdfminer.pdfdocument import PDFTextExtractionNotAllowed
 from pdfminer.pdffont import (PDFCIDFont, PDFFont, PDFType3Font,
-                              PDFUnicodeNotDefined)
+                              PDFUnicodeNotDefined, PDFSimpleFont)
 from pdfminer.pdfpage import PDFPage
 from pdfminer.utils import bbox2str, fsplit, matrix2str
 
@@ -51,8 +51,13 @@ def name2unicode(name):
     """
     if name in glyphname2unicode:
         return glyphname2unicode[name]
-    if name.startswith('g'):
+    if name.startswith('g') or name.startswith('a'):
         raise KeyError(name)
+    if name.startswith('uni'):
+        try:
+            return chr(int(name[3:], 16))
+        except ValueError:  # Not hexadecimal
+            raise KeyError(name)
     m = STRIP_NAME.search(name)
     if not m:
         raise KeyError(name)
@@ -71,6 +76,18 @@ def PDFFont__init__(self, descriptor, widths, default_width=None):
         self.descent = -self.descent
 PDFFont.__init__ = PDFFont__init__
 
+original_PDFSimpleFont_init = PDFSimpleFont.__init__
+def PDFSimpleFont__init__(self, descriptor, widths, spec):
+    # Font encoding is specified either by a name of
+    # built-in encoding or a dictionary that describes
+    # the differences.
+    original_PDFSimpleFont_init(self, descriptor, widths, spec)
+    # pdfminer is incorrect. If there is no ToUnicode and no Encoding, do not
+    # assume Unicode conversion is possible. RM 9.10.2
+    if not self.unicode_map and 'Encoding' not in spec:
+        self.cid2unicode = {}
+    return
+PDFSimpleFont.__init__ = PDFSimpleFont__init__
 #
 # pdfminer patches when creator is PScript5.dll
 #
@@ -196,15 +213,15 @@ def get_page_analysis(infile, pageno, pscript5_mode):
         )
         patcher.start()
 
-    with Path(infile).open('rb') as f:
+    try:
-        page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
+        with Path(infile).open('rb') as f:
-        try:
+            page = PDFPage.get_pages(f, pagenos=[pageno], maxpages=0)
             interp.process_page(next(page))
-        except PDFTextExtractionNotAllowed as e:
+    except PDFTextExtractionNotAllowed:
-            raise EncryptedPdfError()
+        raise EncryptedPdfError()
-        finally:
+    finally:
-            if pscript5_mode:
+        if pscript5_mode:
-                patcher.stop()
+            patcher.stop()
 
     return dev.get_result()
 

tests/resources/README.rst

@@ -139,11 +139,21 @@ licensed under the specified license.
         - @jbarlow83
         - @jbarlow83
         - CC-BY-SA 4.0
+    *   - truetype_font_nomapping.pdf
+        - example of a PDF with an embedded subsetted TrueType font with no Unicode mapping
+        - @jbarlow83
+        - @jbarlow83
+        - CC-BY-SA 4.0
     *   - trivial.pdf
         - smallest possible valid PDF-1.3 with all required fields
         - @jbarlow83
         - @jbarlow83
         - CC-BY-SA 4.0
+    *   - type3_font_nomapping.pdf
+        - example of a PDF with an embedded subsetted TrueType font with no Unicode mapping
+        - @jbarlow83
+        - @jbarlow83
+        - CC-BY-SA 4.0
     *   - vector.pdf
         - a PDF with vector art and text rendered as curves with no fonts
         - @Catscratch

tests/test_pdfinfo.py

@@ -183,3 +183,16 @@ def test_ocr_detection(resources):
     pdf = pdfinfo.PdfInfo(filename)
     assert not pdf[0].has_vector
     assert pdf[0].has_text
+
+
+@pytest.mark.parametrize(
+    'testfile', ('truetype_font_nomapping.pdf', 'type3_font_nomapping.pdf')
+)
+def test_corrupt_font_detection(resources, testfile):
+    filename = resources / testfile
+    with pytest.raises(NotImplementedError):
+        pdf = pdfinfo.PdfInfo(filename)
+        pdf[0].has_corrupt_text
+
+    pdf = pdfinfo.PdfInfo(filename, detailed_page_analysis=True)
+    assert pdf[0].has_corrupt_text