v9.5.0 release notes revised

docs/release_notes.rst

@@ -13,6 +13,12 @@ Note that it is licensed under GPLv3, so scripts that
 ``import ocrmypdf`` and are released publicly should probably also be
 licensed under GPLv3.
 
+v9.5.0
+======
+
+-  Added API functions to measure OCR quality.
+-  Modest improvements to handling PDFs with difficult/non compliant metadata.
+
 v9.4.0
 ======
 

src/ocrmypdf/__init__.py

@@ -23,6 +23,7 @@ from .exceptions import (
     DpiError,
     EncryptedPdfError,
     ExitCode,
+    ExitCodeException,
     InputFileError,
     MissingDependencyError,
     OutputFileAccessError,

src/ocrmypdf/_pipeline.py

@@ -727,47 +727,53 @@ def should_linearize(working_file, context):
 def metadata_fixup(working_file, context):
     output_file = context.get_path('metafix.pdf')
     options = context.options
-    original = pikepdf.open(context.origin)
-    docinfo = get_docinfo(original, options)
-    pdf = pikepdf.open(working_file)
-    with pdf.open_metadata() as meta:
-        meta.load_from_docinfo(docinfo, delete_missing=False)
-        # If xmp:CreateDate is missing, set it to the modify date to
-        # match Ghostscript, for consistency
-        if 'xmp:CreateDate' not in meta:
-            meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
 
-        meta_original = original.open_metadata()
+    def report_on_metadata(missing):
-        not_copied = set(meta_original.keys()) - set(meta.keys())
+        if not missing:
-        if not_copied:
+            return
-            if options.output_type.startswith('pdfa'):
+        if options.output_type.startswith('pdfa'):
-                context.log.warning(
+            context.log.warning(
-                    "Some input metadata could not be copied because it is not "
+                "Some input metadata could not be copied because it is not "
-                    "permitted in PDF/A. You may wish to examine the output "
+                "permitted in PDF/A. You may wish to examine the output "
-                    "PDF's XMP metadata."
+                "PDF's XMP metadata."
-                )
+            )
-                context.log.debug(
+            context.log.debug(
-                    "The following metadata fields were not copied: %r", not_copied
+                "The following metadata fields were not copied: %r", missing
-                )
+            )
-            else:
+        else:
-                context.log.error(
+            context.log.error(
-                    "Some input metadata could not be copied."
+                "Some input metadata could not be copied."
-                    "You may wish to examine the output PDF's XMP metadata."
+                "You may wish to examine the output PDF's XMP metadata."
-                )
+            )
-                context.log.info(
+            context.log.info(
-                    "The following metadata fields were not copied: %r", not_copied
+                "The following metadata fields were not copied: %r", missing
-                )
+            )
-    pdf.save(
+
-        output_file,
+    with pikepdf.open(context.origin) as original, pikepdf.open(working_file) as pdf:
-        compress_streams=True,
+        docinfo = get_docinfo(original, options)
-        preserve_pdfa=True,
+        with pdf.open_metadata() as meta:
-        object_stream_mode=pikepdf.ObjectStreamMode.generate,
+            meta.load_from_docinfo(docinfo, delete_missing=False, raise_failure=False)
-        linearize=(  # Don't linearize if optimize() will be linearizing too
+            # If xmp:CreateDate is missing, set it to the modify date to
-            should_linearize(working_file, context) if options.optimize == 0 else False
+            # match Ghostscript, for consistency
-        ),
+            if 'xmp:CreateDate' not in meta:
-    )
+                meta['xmp:CreateDate'] = meta.get('xmp:ModifyDate', '')
-    original.close()
+
-    pdf.close()
+            meta_original = original.open_metadata()
+            missing = set(meta_original.keys()) - set(meta.keys())
+            report_on_metadata(missing)
+
+        pdf.save(
+            output_file,
+            compress_streams=True,
+            preserve_pdfa=True,
+            object_stream_mode=pikepdf.ObjectStreamMode.generate,
+            linearize=(  # Don't linearize if optimize() will be linearizing too
+                should_linearize(working_file, context)
+                if options.optimize == 0
+                else False
+            ),
+        )
+
     return output_file
 
 

src/ocrmypdf/_sync.py

@@ -381,6 +381,7 @@ def run_pipeline(options, api=False):
             detailed_page_analysis=options.redo_ocr,
             progbar=options.progress_bar,
         )
+
         context = PDFContext(options, work_folder, origin_pdf, pdfinfo)
 
         # Validate options are okay for this pdf

src/ocrmypdf/api.py

@@ -18,11 +18,10 @@
 import logging
 import os
 import sys
-import warnings
 from contextlib import suppress
 from enum import IntEnum
 from pathlib import Path
-from typing import Dict, List, Optional
+from typing import Dict, List
 
 from tqdm import tqdm
 

src/ocrmypdf/quality.py

@@ -0,0 +1,60 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import re
+from typing import Iterable
+
+"""Utilities to measure OCR quality"""
+
+
+class OcrQualityDictionary:
+    """Manages a dictionary for simple OCR quality checks."""
+
+    def __init__(self, *, wordlist: Iterable[str] = []):
+        """Construct a dictionary from a list of words.
+
+        Words for which capitalization is important should be capitalized in the
+        dictionary. Words that contain spaces or other punctuation will never match.
+        """
+        self.dictionary = set()
+        self.dictionary.update(w for w in wordlist)
+
+    def measure_words_matched(self, ocr_text: str) -> float:
+        """Check how many unique words in the OCR text match a dictionary.
+
+        Words with mixed capitalized are only considered a match if the test word
+        matches that capitalization.
+
+        Returns:
+            number of words that match / number
+        """
+        text = re.sub(r"[0-9_]+", ' ', ocr_text)
+        text = re.sub(r'\W+', ' ', text)
+        text_words_list = re.split(r'\s+', text)
+        text_words = {w for w in text_words_list if len(w) >= 3}
+
+        matches = 0
+        for w in text_words:
+            if w in self.dictionary or (
+                w != w.lower() and w.lower() in self.dictionary
+            ):
+                matches += 1
+        if matches > 0:
+            hit_ratio = matches / len(text_words)
+        else:
+            hit_ratio = 0.0
+        return hit_ratio

tests/test_quality.py

@@ -0,0 +1,35 @@
+# © 2020 James R. Barlow: github.com/jbarlow83
+#
+# This file is part of OCRmyPDF.
+#
+# OCRmyPDF is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# OCRmyPDF is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with OCRmyPDF.  If not, see <http://www.gnu.org/licenses/>.
+
+import pytest
+
+import ocrmypdf.quality as qual
+
+
+def test_quality_measurement():
+    oqd = qual.OcrQualityDictionary(
+        wordlist=["words", "words", "quick", "brown", "fox", "dog", "lazy"]
+    )
+    assert len(oqd.dictionary) == 6  # 6 unique
+
+    assert (
+        oqd.measure_words_matched("The quick brown fox jumps quickly over the lazy dog")
+        == 0.5
+    )
+    assert oqd.measure_words_matched("12345 10% _f  7fox -brown   | words") == 1.0
+
+    assert oqd.measure_words_matched("quick quick quick") == 1.0